src/intel/vulkan/genX_cmd_buffer.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26
  27 #include "anv_private.h"
  28
  29 #include "genxml/gen_macros.h"
  30 #include "genxml/genX_pack.h"
  31
  32 void
  33 genX(cmd_buffer_emit_state_base_address)(struct anv_cmd_buffer *cmd_buffer)
  34 {
  35    struct anv_device *device = cmd_buffer->device;
  36    struct anv_bo *scratch_bo = NULL;
  37
  38    cmd_buffer->state.scratch_size =
  39       anv_block_pool_size(&device->scratch_block_pool);
  40    if (cmd_buffer->state.scratch_size > 0)
  41       scratch_bo = &device->scratch_block_pool.bo;
  42
  43 /* XXX: Do we need this on more than just BDW? */
  44 #if (GEN_GEN >= 8)
  45    /* Emit a render target cache flush.
  46     *
  47     * This isn't documented anywhere in the PRM.  However, it seems to be
  48     * necessary prior to changing the surface state base adress.  Without
  49     * this, we get GPU hangs when using multi-level command buffers which
  50     * clear depth, reset state base address, and then go render stuff.
  51     */
  52    anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
  53                   .RenderTargetCacheFlushEnable = true);
  54 #endif
  55
  56    anv_batch_emit(&cmd_buffer->batch, GENX(STATE_BASE_ADDRESS),
  57       .GeneralStateBaseAddress = { scratch_bo, 0 },
  58       .GeneralStateMemoryObjectControlState = GENX(MOCS),
  59       .GeneralStateBaseAddressModifyEnable = true,
  60
  61       .SurfaceStateBaseAddress = anv_cmd_buffer_surface_base_address(cmd_buffer),
  62       .SurfaceStateMemoryObjectControlState = GENX(MOCS),
  63       .SurfaceStateBaseAddressModifyEnable = true,
  64
  65       .DynamicStateBaseAddress = { &device->dynamic_state_block_pool.bo, 0 },
  66       .DynamicStateMemoryObjectControlState = GENX(MOCS),
  67       .DynamicStateBaseAddressModifyEnable = true,
  68
  69       .IndirectObjectBaseAddress = { NULL, 0 },
  70       .IndirectObjectMemoryObjectControlState = GENX(MOCS),
  71       .IndirectObjectBaseAddressModifyEnable = true,
  72
  73       .InstructionBaseAddress = { &device->instruction_block_pool.bo, 0 },
  74       .InstructionMemoryObjectControlState = GENX(MOCS),
  75       .InstructionBaseAddressModifyEnable = true,
  76
  77 #  if (GEN_GEN >= 8)
  78       /* Broadwell requires that we specify a buffer size for a bunch of
  79        * these fields.  However, since we will be growing the BO's live, we
  80        * just set them all to the maximum.
  81        */
  82       .GeneralStateBufferSize = 0xfffff,
  83       .GeneralStateBufferSizeModifyEnable = true,
  84       .DynamicStateBufferSize = 0xfffff,
  85       .DynamicStateBufferSizeModifyEnable = true,
  86       .IndirectObjectBufferSize = 0xfffff,
  87       .IndirectObjectBufferSizeModifyEnable = true,
  88       .InstructionBufferSize = 0xfffff,
  89       .InstructionBuffersizeModifyEnable = true,
  90 #  endif
  91    );
  92
  93    /* After re-setting the surface state base address, we have to do some
  94     * cache flusing so that the sampler engine will pick up the new
  95     * SURFACE_STATE objects and binding tables. From the Broadwell PRM,
  96     * Shared Function > 3D Sampler > State > State Caching (page 96):
  97     *
  98     *    Coherency with system memory in the state cache, like the texture
  99     *    cache is handled partially by software. It is expected that the
 100     *    command stream or shader will issue Cache Flush operation or
 101     *    Cache_Flush sampler message to ensure that the L1 cache remains
 102     *    coherent with system memory.
 103     *
 104     *    [...]
 105     *
 106     *    Whenever the value of the Dynamic_State_Base_Addr,
 107     *    Surface_State_Base_Addr are altered, the L1 state cache must be
 108     *    invalidated to ensure the new surface or sampler state is fetched
 109     *    from system memory.
 110     *
 111     * The PIPE_CONTROL command has a "State Cache Invalidation Enable" bit
 112     * which, according the PIPE_CONTROL instruction documentation in the
 113     * Broadwell PRM:
 114     *
 115     *    Setting this bit is independent of any other bit in this packet.
 116     *    This bit controls the invalidation of the L1 and L2 state caches
 117     *    at the top of the pipe i.e. at the parsing time.
 118     *
 119     * Unfortunately, experimentation seems to indicate that state cache
 120     * invalidation through a PIPE_CONTROL does nothing whatsoever in
 121     * regards to surface state and binding tables.  In stead, it seems that
 122     * invalidating the texture cache is what is actually needed.
 123     *
 124     * XXX:  As far as we have been able to determine through
 125     * experimentation, shows that flush the texture cache appears to be
 126     * sufficient.  The theory here is that all of the sampling/rendering
 127     * units cache the binding table in the texture cache.  However, we have
 128     * yet to be able to actually confirm this.
 129     */
 130    anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
 131                   .TextureCacheInvalidationEnable = true);
 132 }
 133
 134 void genX(CmdPipelineBarrier)(
 135     VkCommandBuffer                             commandBuffer,
 136     VkPipelineStageFlags                        srcStageMask,
 137     VkPipelineStageFlags                        destStageMask,
 138     VkBool32                                    byRegion,
 139     uint32_t                                    memoryBarrierCount,
 140     const VkMemoryBarrier*                      pMemoryBarriers,
 141     uint32_t                                    bufferMemoryBarrierCount,
 142     const VkBufferMemoryBarrier*                pBufferMemoryBarriers,
 143     uint32_t                                    imageMemoryBarrierCount,
 144     const VkImageMemoryBarrier*                 pImageMemoryBarriers)
 145 {
 146    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 147    uint32_t b, *dw;
 148
 149    /* XXX: Right now, we're really dumb and just flush whatever categories
 150     * the app asks for.  One of these days we may make this a bit better
 151     * but right now that's all the hardware allows for in most areas.
 152     */
 153    VkAccessFlags src_flags = 0;
 154    VkAccessFlags dst_flags = 0;
 155
 156    for (uint32_t i = 0; i < memoryBarrierCount; i++) {
 157       src_flags |= pMemoryBarriers[i].srcAccessMask;
 158       dst_flags |= pMemoryBarriers[i].dstAccessMask;
 159    }
 160
 161    for (uint32_t i = 0; i < bufferMemoryBarrierCount; i++) {
 162       src_flags |= pBufferMemoryBarriers[i].srcAccessMask;
 163       dst_flags |= pBufferMemoryBarriers[i].dstAccessMask;
 164    }
 165
 166    for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) {
 167       src_flags |= pImageMemoryBarriers[i].srcAccessMask;
 168       dst_flags |= pImageMemoryBarriers[i].dstAccessMask;
 169    }
 170
 171    /* Mask out the Source access flags we care about */
 172    const uint32_t src_mask =
 173       VK_ACCESS_SHADER_WRITE_BIT |
 174       VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
 175       VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT |
 176       VK_ACCESS_TRANSFER_WRITE_BIT;
 177
 178    src_flags = src_flags & src_mask;
 179
 180    /* Mask out the destination access flags we care about */
 181    const uint32_t dst_mask =
 182       VK_ACCESS_INDIRECT_COMMAND_READ_BIT |
 183       VK_ACCESS_INDEX_READ_BIT |
 184       VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT |
 185       VK_ACCESS_UNIFORM_READ_BIT |
 186       VK_ACCESS_SHADER_READ_BIT |
 187       VK_ACCESS_COLOR_ATTACHMENT_READ_BIT |
 188       VK_ACCESS_TRANSFER_READ_BIT;
 189
 190    dst_flags = dst_flags & dst_mask;
 191
 192    /* The src flags represent how things were used previously.  This is
 193     * what we use for doing flushes.
 194     */
 195    struct GENX(PIPE_CONTROL) flush_cmd = {
 196       GENX(PIPE_CONTROL_header),
 197       .PostSyncOperation = NoWrite,
 198    };
 199
 200    for_each_bit(b, src_flags) {
 201       switch ((VkAccessFlagBits)(1 << b)) {
 202       case VK_ACCESS_SHADER_WRITE_BIT:
 203          flush_cmd.DCFlushEnable = true;
 204          break;
 205       case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT:
 206          flush_cmd.RenderTargetCacheFlushEnable = true;
 207          break;
 208       case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT:
 209          flush_cmd.DepthCacheFlushEnable = true;
 210          break;
 211       case VK_ACCESS_TRANSFER_WRITE_BIT:
 212          flush_cmd.RenderTargetCacheFlushEnable = true;
 213          flush_cmd.DepthCacheFlushEnable = true;
 214          break;
 215       default:
 216          unreachable("should've masked this out by now");
 217       }
 218    }
 219
 220    /* If we end up doing two PIPE_CONTROLs, the first, flusing one also has to
 221     * stall and wait for the flushing to finish, so we don't re-dirty the
 222     * caches with in-flight rendering after the second PIPE_CONTROL
 223     * invalidates.
 224     */
 225
 226    if (dst_flags)
 227       flush_cmd.CommandStreamerStallEnable = true;
 228
 229    if (src_flags && dst_flags) {
 230       dw = anv_batch_emit_dwords(&cmd_buffer->batch, GENX(PIPE_CONTROL_length));
 231       GENX(PIPE_CONTROL_pack)(&cmd_buffer->batch, dw, &flush_cmd);
 232    }
 233
 234    /* The dst flags represent how things will be used in the future.  This
 235     * is what we use for doing cache invalidations.
 236     */
 237    struct GENX(PIPE_CONTROL) invalidate_cmd = {
 238       GENX(PIPE_CONTROL_header),
 239       .PostSyncOperation = NoWrite,
 240    };
 241
 242    for_each_bit(b, dst_flags) {
 243       switch ((VkAccessFlagBits)(1 << b)) {
 244       case VK_ACCESS_INDIRECT_COMMAND_READ_BIT:
 245       case VK_ACCESS_INDEX_READ_BIT:
 246       case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT:
 247          invalidate_cmd.VFCacheInvalidationEnable = true;
 248          break;
 249       case VK_ACCESS_UNIFORM_READ_BIT:
 250          invalidate_cmd.ConstantCacheInvalidationEnable = true;
 251          /* fallthrough */
 252       case VK_ACCESS_SHADER_READ_BIT:
 253          invalidate_cmd.TextureCacheInvalidationEnable = true;
 254          break;
 255       case VK_ACCESS_COLOR_ATTACHMENT_READ_BIT:
 256          invalidate_cmd.TextureCacheInvalidationEnable = true;
 257          break;
 258       case VK_ACCESS_TRANSFER_READ_BIT:
 259          invalidate_cmd.TextureCacheInvalidationEnable = true;
 260          break;
 261       default:
 262          unreachable("should've masked this out by now");
 263       }
 264    }
 265
 266    if (dst_flags) {
 267       dw = anv_batch_emit_dwords(&cmd_buffer->batch, GENX(PIPE_CONTROL_length));
 268       GENX(PIPE_CONTROL_pack)(&cmd_buffer->batch, dw, &invalidate_cmd);
 269    }
 270 }
 271
 272 static void
 273 emit_base_vertex_instance_bo(struct anv_cmd_buffer *cmd_buffer,
 274                              struct anv_bo *bo, uint32_t offset)
 275 {
 276    uint32_t *p = anv_batch_emitn(&cmd_buffer->batch, 5,
 277                                  GENX(3DSTATE_VERTEX_BUFFERS));
 278
 279    GENX(VERTEX_BUFFER_STATE_pack)(&cmd_buffer->batch, p + 1,
 280       &(struct GENX(VERTEX_BUFFER_STATE)) {
 281          .VertexBufferIndex = 32, /* Reserved for this */
 282          .AddressModifyEnable = true,
 283          .BufferPitch = 0,
 284 #if (GEN_GEN >= 8)
 285          .MemoryObjectControlState = GENX(MOCS),
 286          .BufferStartingAddress = { bo, offset },
 287          .BufferSize = 8
 288 #else
 289          .VertexBufferMemoryObjectControlState = GENX(MOCS),
 290          .BufferStartingAddress = { bo, offset },
 291          .EndAddress = { bo, offset + 8 },
 292 #endif
 293       });
 294 }
 295
 296 static void
 297 emit_base_vertex_instance(struct anv_cmd_buffer *cmd_buffer,
 298                           uint32_t base_vertex, uint32_t base_instance)
 299 {
 300    struct anv_state id_state =
 301       anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, 8, 4);
 302
 303    ((uint32_t *)id_state.map)[0] = base_vertex;
 304    ((uint32_t *)id_state.map)[1] = base_instance;
 305
 306    if (!cmd_buffer->device->info.has_llc)
 307       anv_state_clflush(id_state);
 308
 309    emit_base_vertex_instance_bo(cmd_buffer,
 310       &cmd_buffer->device->dynamic_state_block_pool.bo, id_state.offset);
 311 }
 312
 313 void genX(CmdDraw)(
 314     VkCommandBuffer                             commandBuffer,
 315     uint32_t                                    vertexCount,
 316     uint32_t                                    instanceCount,
 317     uint32_t                                    firstVertex,
 318     uint32_t                                    firstInstance)
 319 {
 320    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 321    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 322
 323    genX(cmd_buffer_flush_state)(cmd_buffer);
 324
 325    if (cmd_buffer->state.pipeline->vs_prog_data.uses_basevertex ||
 326        cmd_buffer->state.pipeline->vs_prog_data.uses_baseinstance)
 327       emit_base_vertex_instance(cmd_buffer, firstVertex, firstInstance);
 328
 329    anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
 330       .VertexAccessType                         = SEQUENTIAL,
 331       .PrimitiveTopologyType                    = pipeline->topology,
 332       .VertexCountPerInstance                   = vertexCount,
 333       .StartVertexLocation                      = firstVertex,
 334       .InstanceCount                            = instanceCount,
 335       .StartInstanceLocation                    = firstInstance,
 336       .BaseVertexLocation                       = 0);
 337 }
 338
 339 void genX(CmdDrawIndexed)(
 340     VkCommandBuffer                             commandBuffer,
 341     uint32_t                                    indexCount,
 342     uint32_t                                    instanceCount,
 343     uint32_t                                    firstIndex,
 344     int32_t                                     vertexOffset,
 345     uint32_t                                    firstInstance)
 346 {
 347    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 348    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 349
 350    genX(cmd_buffer_flush_state)(cmd_buffer);
 351
 352    if (cmd_buffer->state.pipeline->vs_prog_data.uses_basevertex ||
 353        cmd_buffer->state.pipeline->vs_prog_data.uses_baseinstance)
 354       emit_base_vertex_instance(cmd_buffer, vertexOffset, firstInstance);
 355
 356    anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
 357       .VertexAccessType                         = RANDOM,
 358       .PrimitiveTopologyType                    = pipeline->topology,
 359       .VertexCountPerInstance                   = indexCount,
 360       .StartVertexLocation                      = firstIndex,
 361       .InstanceCount                            = instanceCount,
 362       .StartInstanceLocation                    = firstInstance,
 363       .BaseVertexLocation                       = vertexOffset);
 364 }
 365
 366 /* Auto-Draw / Indirect Registers */
 367 #define GEN7_3DPRIM_END_OFFSET          0x2420
 368 #define GEN7_3DPRIM_START_VERTEX        0x2430
 369 #define GEN7_3DPRIM_VERTEX_COUNT        0x2434
 370 #define GEN7_3DPRIM_INSTANCE_COUNT      0x2438
 371 #define GEN7_3DPRIM_START_INSTANCE      0x243C
 372 #define GEN7_3DPRIM_BASE_VERTEX         0x2440
 373
 374 static void
 375 emit_lrm(struct anv_batch *batch,
 376          uint32_t reg, struct anv_bo *bo, uint32_t offset)
 377 {
 378    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM),
 379                   .RegisterAddress = reg,
 380                   .MemoryAddress = { bo, offset });
 381 }
 382
 383 static void
 384 emit_lri(struct anv_batch *batch, uint32_t reg, uint32_t imm)
 385 {
 386    anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM),
 387                   .RegisterOffset = reg,
 388                   .DataDWord = imm);
 389 }
 390
 391 void genX(CmdDrawIndirect)(
 392     VkCommandBuffer                             commandBuffer,
 393     VkBuffer                                    _buffer,
 394     VkDeviceSize                                offset,
 395     uint32_t                                    drawCount,
 396     uint32_t                                    stride)
 397 {
 398    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 399    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 400    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 401    struct anv_bo *bo = buffer->bo;
 402    uint32_t bo_offset = buffer->offset + offset;
 403
 404    genX(cmd_buffer_flush_state)(cmd_buffer);
 405
 406    if (cmd_buffer->state.pipeline->vs_prog_data.uses_basevertex ||
 407        cmd_buffer->state.pipeline->vs_prog_data.uses_baseinstance)
 408       emit_base_vertex_instance_bo(cmd_buffer, bo, bo_offset + 8);
 409
 410    emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
 411    emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
 412    emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
 413    emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 12);
 414    emit_lri(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, 0);
 415
 416    anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
 417       .IndirectParameterEnable                  = true,
 418       .VertexAccessType                         = SEQUENTIAL,
 419       .PrimitiveTopologyType                    = pipeline->topology);
 420 }
 421
 422 void genX(CmdDrawIndexedIndirect)(
 423     VkCommandBuffer                             commandBuffer,
 424     VkBuffer                                    _buffer,
 425     VkDeviceSize                                offset,
 426     uint32_t                                    drawCount,
 427     uint32_t                                    stride)
 428 {
 429    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 430    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 431    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 432    struct anv_bo *bo = buffer->bo;
 433    uint32_t bo_offset = buffer->offset + offset;
 434
 435    genX(cmd_buffer_flush_state)(cmd_buffer);
 436
 437    /* TODO: We need to stomp base vertex to 0 somehow */
 438    if (cmd_buffer->state.pipeline->vs_prog_data.uses_basevertex ||
 439        cmd_buffer->state.pipeline->vs_prog_data.uses_baseinstance)
 440       emit_base_vertex_instance_bo(cmd_buffer, bo, bo_offset + 12);
 441
 442    emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
 443    emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
 444    emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
 445    emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, bo, bo_offset + 12);
 446    emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 16);
 447
 448    anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
 449       .IndirectParameterEnable                  = true,
 450       .VertexAccessType                         = RANDOM,
 451       .PrimitiveTopologyType                    = pipeline->topology);
 452 }
 453
 454
 455 void genX(CmdDispatch)(
 456     VkCommandBuffer                             commandBuffer,
 457     uint32_t                                    x,
 458     uint32_t                                    y,
 459     uint32_t                                    z)
 460 {
 461    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 462    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 463    struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
 464
 465    if (prog_data->uses_num_work_groups) {
 466       struct anv_state state =
 467          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, 12, 4);
 468       uint32_t *sizes = state.map;
 469       sizes[0] = x;
 470       sizes[1] = y;
 471       sizes[2] = z;
 472       if (!cmd_buffer->device->info.has_llc)
 473          anv_state_clflush(state);
 474       cmd_buffer->state.num_workgroups_offset = state.offset;
 475       cmd_buffer->state.num_workgroups_bo =
 476          &cmd_buffer->device->dynamic_state_block_pool.bo;
 477    }
 478
 479    genX(cmd_buffer_flush_compute_state)(cmd_buffer);
 480
 481    anv_batch_emit(&cmd_buffer->batch, GENX(GPGPU_WALKER),
 482                   .SIMDSize = prog_data->simd_size / 16,
 483                   .ThreadDepthCounterMaximum = 0,
 484                   .ThreadHeightCounterMaximum = 0,
 485                   .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max - 1,
 486                   .ThreadGroupIDXDimension = x,
 487                   .ThreadGroupIDYDimension = y,
 488                   .ThreadGroupIDZDimension = z,
 489                   .RightExecutionMask = pipeline->cs_right_mask,
 490                   .BottomExecutionMask = 0xffffffff);
 491
 492    anv_batch_emit(&cmd_buffer->batch, GENX(MEDIA_STATE_FLUSH));
 493 }
 494
 495 #define GPGPU_DISPATCHDIMX 0x2500
 496 #define GPGPU_DISPATCHDIMY 0x2504
 497 #define GPGPU_DISPATCHDIMZ 0x2508
 498
 499 #define MI_PREDICATE_SRC0  0x2400
 500 #define MI_PREDICATE_SRC1  0x2408
 501
 502 void genX(CmdDispatchIndirect)(
 503     VkCommandBuffer                             commandBuffer,
 504     VkBuffer                                    _buffer,
 505     VkDeviceSize                                offset)
 506 {
 507    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 508    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 509    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 510    struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
 511    struct anv_bo *bo = buffer->bo;
 512    uint32_t bo_offset = buffer->offset + offset;
 513    struct anv_batch *batch = &cmd_buffer->batch;
 514
 515    if (prog_data->uses_num_work_groups) {
 516       cmd_buffer->state.num_workgroups_offset = bo_offset;
 517       cmd_buffer->state.num_workgroups_bo = bo;
 518    }
 519
 520    genX(cmd_buffer_flush_compute_state)(cmd_buffer);
 521
 522    emit_lrm(batch, GPGPU_DISPATCHDIMX, bo, bo_offset);
 523    emit_lrm(batch, GPGPU_DISPATCHDIMY, bo, bo_offset + 4);
 524    emit_lrm(batch, GPGPU_DISPATCHDIMZ, bo, bo_offset + 8);
 525
 526 #if GEN_GEN <= 7
 527    /* Clear upper 32-bits of SRC0 and all 64-bits of SRC1 */
 528    emit_lri(batch, MI_PREDICATE_SRC0 + 4, 0);
 529    emit_lri(batch, MI_PREDICATE_SRC1 + 0, 0);
 530    emit_lri(batch, MI_PREDICATE_SRC1 + 4, 0);
 531
 532    /* Load compute_dispatch_indirect_x_size into SRC0 */
 533    emit_lrm(batch, MI_PREDICATE_SRC0, bo, bo_offset + 0);
 534
 535    /* predicate = (compute_dispatch_indirect_x_size == 0); */
 536    anv_batch_emit(batch, GENX(MI_PREDICATE),
 537                   .LoadOperation = LOAD_LOAD,
 538                   .CombineOperation = COMBINE_SET,
 539                   .CompareOperation = COMPARE_SRCS_EQUAL);
 540
 541    /* Load compute_dispatch_indirect_y_size into SRC0 */
 542    emit_lrm(batch, MI_PREDICATE_SRC0, bo, bo_offset + 4);
 543
 544    /* predicate |= (compute_dispatch_indirect_y_size == 0); */
 545    anv_batch_emit(batch, GENX(MI_PREDICATE),
 546                   .LoadOperation = LOAD_LOAD,
 547                   .CombineOperation = COMBINE_OR,
 548                   .CompareOperation = COMPARE_SRCS_EQUAL);
 549
 550    /* Load compute_dispatch_indirect_z_size into SRC0 */
 551    emit_lrm(batch, MI_PREDICATE_SRC0, bo, bo_offset + 8);
 552
 553    /* predicate |= (compute_dispatch_indirect_z_size == 0); */
 554    anv_batch_emit(batch, GENX(MI_PREDICATE),
 555                   .LoadOperation = LOAD_LOAD,
 556                   .CombineOperation = COMBINE_OR,
 557                   .CompareOperation = COMPARE_SRCS_EQUAL);
 558
 559    /* predicate = !predicate; */
 560 #define COMPARE_FALSE                           1
 561    anv_batch_emit(batch, GENX(MI_PREDICATE),
 562                   .LoadOperation = LOAD_LOADINV,
 563                   .CombineOperation = COMBINE_OR,
 564                   .CompareOperation = COMPARE_FALSE);
 565 #endif
 566
 567    anv_batch_emit(batch, GENX(GPGPU_WALKER),
 568                   .IndirectParameterEnable = true,
 569                   .PredicateEnable = GEN_GEN <= 7,
 570                   .SIMDSize = prog_data->simd_size / 16,
 571                   .ThreadDepthCounterMaximum = 0,
 572                   .ThreadHeightCounterMaximum = 0,
 573                   .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max - 1,
 574                   .RightExecutionMask = pipeline->cs_right_mask,
 575                   .BottomExecutionMask = 0xffffffff);
 576
 577    anv_batch_emit(batch, GENX(MEDIA_STATE_FLUSH));
 578 }
 579
 580 void
 581 genX(flush_pipeline_select_3d)(struct anv_cmd_buffer *cmd_buffer)
 582 {
 583    if (cmd_buffer->state.current_pipeline != _3D) {
 584       anv_batch_emit(&cmd_buffer->batch, GENX(PIPELINE_SELECT),
 585 #if GEN_GEN >= 9
 586                      .MaskBits = 3,
 587 #endif
 588                      .PipelineSelection = _3D);
 589       cmd_buffer->state.current_pipeline = _3D;
 590    }
 591 }
 592
 593 static void
 594 cmd_buffer_emit_depth_stencil(struct anv_cmd_buffer *cmd_buffer)
 595 {
 596    struct anv_device *device = cmd_buffer->device;
 597    const struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
 598    const struct anv_image_view *iview =
 599       anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
 600    const struct anv_image *image = iview ? iview->image : NULL;
 601    const struct anv_format *anv_format =
 602       iview ? anv_format_for_vk_format(iview->vk_format) : NULL;
 603    const bool has_depth = iview && anv_format->has_depth;
 604    const bool has_stencil = iview && anv_format->has_stencil;
 605
 606    /* FIXME: Implement the PMA stall W/A */
 607    /* FIXME: Width and Height are wrong */
 608
 609    /* Emit 3DSTATE_DEPTH_BUFFER */
 610    if (has_depth) {
 611       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
 612          .SurfaceType = SURFTYPE_2D,
 613          .DepthWriteEnable = true,
 614          .StencilWriteEnable = has_stencil,
 615          .HierarchicalDepthBufferEnable = false,
 616          .SurfaceFormat = isl_surf_get_depth_format(&device->isl_dev,
 617                                                     &image->depth_surface.isl),
 618          .SurfacePitch = image->depth_surface.isl.row_pitch - 1,
 619          .SurfaceBaseAddress = {
 620             .bo = image->bo,
 621             .offset = image->offset + image->depth_surface.offset,
 622          },
 623          .Height = fb->height - 1,
 624          .Width = fb->width - 1,
 625          .LOD = 0,
 626          .Depth = 1 - 1,
 627          .MinimumArrayElement = 0,
 628          .DepthBufferObjectControlState = GENX(MOCS),
 629 #if GEN_GEN >= 8
 630          .SurfaceQPitch = isl_surf_get_array_pitch_el_rows(&image->depth_surface.isl) >> 2,
 631 #endif
 632          .RenderTargetViewExtent = 1 - 1);
 633    } else {
 634       /* Even when no depth buffer is present, the hardware requires that
 635        * 3DSTATE_DEPTH_BUFFER be programmed correctly. The Broadwell PRM says:
 636        *
 637        *    If a null depth buffer is bound, the driver must instead bind depth as:
 638        *       3DSTATE_DEPTH.SurfaceType = SURFTYPE_2D
 639        *       3DSTATE_DEPTH.Width = 1
 640        *       3DSTATE_DEPTH.Height = 1
 641        *       3DSTATE_DEPTH.SuraceFormat = D16_UNORM
 642        *       3DSTATE_DEPTH.SurfaceBaseAddress = 0
 643        *       3DSTATE_DEPTH.HierarchicalDepthBufferEnable = 0
 644        *       3DSTATE_WM_DEPTH_STENCIL.DepthTestEnable = 0
 645        *       3DSTATE_WM_DEPTH_STENCIL.DepthBufferWriteEnable = 0
 646        *
 647        * The PRM is wrong, though. The width and height must be programmed to
 648        * actual framebuffer's width and height, even when neither depth buffer
 649        * nor stencil buffer is present.
 650        */
 651       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
 652          .SurfaceType = SURFTYPE_2D,
 653          .SurfaceFormat = D16_UNORM,
 654          .Width = fb->width - 1,
 655          .Height = fb->height - 1,
 656          .StencilWriteEnable = has_stencil);
 657    }
 658
 659    /* Emit 3DSTATE_STENCIL_BUFFER */
 660    if (has_stencil) {
 661       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER),
 662 #if GEN_GEN >= 8 || GEN_IS_HASWELL
 663          .StencilBufferEnable = true,
 664 #endif
 665          .StencilBufferObjectControlState = GENX(MOCS),
 666
 667          /* Stencil buffers have strange pitch. The PRM says:
 668           *
 669           *    The pitch must be set to 2x the value computed based on width,
 670           *    as the stencil buffer is stored with two rows interleaved.
 671           */
 672          .SurfacePitch = 2 * image->stencil_surface.isl.row_pitch - 1,
 673
 674 #if GEN_GEN >= 8
 675          .SurfaceQPitch = isl_surf_get_array_pitch_el_rows(&image->stencil_surface.isl) >> 2,
 676 #endif
 677          .SurfaceBaseAddress = {
 678             .bo = image->bo,
 679             .offset = image->offset + image->stencil_surface.offset,
 680          });
 681    } else {
 682       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER));
 683    }
 684
 685    /* Disable hierarchial depth buffers. */
 686    anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_HIER_DEPTH_BUFFER));
 687
 688    /* Clear the clear params. */
 689    anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CLEAR_PARAMS));
 690 }
 691
 692 /**
 693  * @see anv_cmd_buffer_set_subpass()
 694  */
 695 void
 696 genX(cmd_buffer_set_subpass)(struct anv_cmd_buffer *cmd_buffer,
 697                              struct anv_subpass *subpass)
 698 {
 699    cmd_buffer->state.subpass = subpass;
 700
 701    cmd_buffer->state.descriptors_dirty |= VK_SHADER_STAGE_FRAGMENT_BIT;
 702
 703    cmd_buffer_emit_depth_stencil(cmd_buffer);
 704 }
 705
 706 void genX(CmdBeginRenderPass)(
 707     VkCommandBuffer                             commandBuffer,
 708     const VkRenderPassBeginInfo*                pRenderPassBegin,
 709     VkSubpassContents                           contents)
 710 {
 711    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 712    ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
 713    ANV_FROM_HANDLE(anv_framebuffer, framebuffer, pRenderPassBegin->framebuffer);
 714
 715    cmd_buffer->state.framebuffer = framebuffer;
 716    cmd_buffer->state.pass = pass;
 717    anv_cmd_state_setup_attachments(cmd_buffer, pRenderPassBegin);
 718
 719    genX(flush_pipeline_select_3d)(cmd_buffer);
 720
 721    const VkRect2D *render_area = &pRenderPassBegin->renderArea;
 722
 723    anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DRAWING_RECTANGLE),
 724                   .ClippedDrawingRectangleYMin = render_area->offset.y,
 725                   .ClippedDrawingRectangleXMin = render_area->offset.x,
 726                   .ClippedDrawingRectangleYMax =
 727                      render_area->offset.y + render_area->extent.height - 1,
 728                   .ClippedDrawingRectangleXMax =
 729                      render_area->offset.x + render_area->extent.width - 1,
 730                   .DrawingRectangleOriginY = 0,
 731                   .DrawingRectangleOriginX = 0);
 732
 733    genX(cmd_buffer_set_subpass)(cmd_buffer, pass->subpasses);
 734    anv_cmd_buffer_clear_subpass(cmd_buffer);
 735 }
 736
 737 void genX(CmdNextSubpass)(
 738     VkCommandBuffer                             commandBuffer,
 739     VkSubpassContents                           contents)
 740 {
 741    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 742
 743    assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
 744
 745    anv_cmd_buffer_resolve_subpass(cmd_buffer);
 746    genX(cmd_buffer_set_subpass)(cmd_buffer, cmd_buffer->state.subpass + 1);
 747    anv_cmd_buffer_clear_subpass(cmd_buffer);
 748 }
 749
 750 void genX(CmdEndRenderPass)(
 751     VkCommandBuffer                             commandBuffer)
 752 {
 753    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 754
 755    anv_cmd_buffer_resolve_subpass(cmd_buffer);
 756 }