src/vulkan/gen7_cmd_buffer.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include <fcntl.h>
  29
  30 #include "anv_private.h"
  31
  32 #include "gen7_pack.h"
  33 #include "gen75_pack.h"
  34
  35 static void
  36 cmd_buffer_flush_push_constants(struct anv_cmd_buffer *cmd_buffer)
  37 {
  38    static const uint32_t push_constant_opcodes[] = {
  39       [VK_SHADER_STAGE_VERTEX]                  = 21,
  40       [VK_SHADER_STAGE_TESS_CONTROL]            = 25, /* HS */
  41       [VK_SHADER_STAGE_TESS_EVALUATION]         = 26, /* DS */
  42       [VK_SHADER_STAGE_GEOMETRY]                = 22,
  43       [VK_SHADER_STAGE_FRAGMENT]                = 23,
  44       [VK_SHADER_STAGE_COMPUTE]                 = 0,
  45    };
  46
  47    VkShaderStage stage;
  48    VkShaderStageFlags flushed = 0;
  49
  50    for_each_bit(stage, cmd_buffer->state.push_constants_dirty) {
  51       struct anv_state state = anv_cmd_buffer_push_constants(cmd_buffer, stage);
  52
  53       if (state.offset == 0)
  54          continue;
  55
  56       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_CONSTANT_VS,
  57                      ._3DCommandSubOpcode = push_constant_opcodes[stage],
  58                      .ConstantBody = {
  59                         .PointerToConstantBuffer0 = { .offset = state.offset },
  60                         .ConstantBuffer0ReadLength = DIV_ROUND_UP(state.alloc_size, 32),
  61                      });
  62
  63       flushed |= 1 << stage;
  64    }
  65
  66    cmd_buffer->state.push_constants_dirty &= ~flushed;
  67 }
  68
  69 static VkResult
  70 flush_descriptor_set(struct anv_cmd_buffer *cmd_buffer, VkShaderStage stage)
  71 {
  72    struct anv_state surfaces = { 0, }, samplers = { 0, };
  73    VkResult result;
  74
  75    result = anv_cmd_buffer_emit_samplers(cmd_buffer, stage, &samplers);
  76    if (result != VK_SUCCESS)
  77       return result;
  78    result = anv_cmd_buffer_emit_binding_table(cmd_buffer, stage, &surfaces);
  79    if (result != VK_SUCCESS)
  80       return result;
  81
  82    static const uint32_t sampler_state_opcodes[] = {
  83       [VK_SHADER_STAGE_VERTEX]                  = 43,
  84       [VK_SHADER_STAGE_TESS_CONTROL]            = 44, /* HS */
  85       [VK_SHADER_STAGE_TESS_EVALUATION]         = 45, /* DS */
  86       [VK_SHADER_STAGE_GEOMETRY]                = 46,
  87       [VK_SHADER_STAGE_FRAGMENT]                = 47,
  88       [VK_SHADER_STAGE_COMPUTE]                 = 0,
  89    };
  90
  91    static const uint32_t binding_table_opcodes[] = {
  92       [VK_SHADER_STAGE_VERTEX]                  = 38,
  93       [VK_SHADER_STAGE_TESS_CONTROL]            = 39,
  94       [VK_SHADER_STAGE_TESS_EVALUATION]         = 40,
  95       [VK_SHADER_STAGE_GEOMETRY]                = 41,
  96       [VK_SHADER_STAGE_FRAGMENT]                = 42,
  97       [VK_SHADER_STAGE_COMPUTE]                 = 0,
  98    };
  99
 100    if (samplers.alloc_size > 0) {
 101       anv_batch_emit(&cmd_buffer->batch,
 102                      GEN7_3DSTATE_SAMPLER_STATE_POINTERS_VS,
 103                      ._3DCommandSubOpcode  = sampler_state_opcodes[stage],
 104                      .PointertoVSSamplerState = samplers.offset);
 105    }
 106
 107    if (surfaces.alloc_size > 0) {
 108       anv_batch_emit(&cmd_buffer->batch,
 109                      GEN7_3DSTATE_BINDING_TABLE_POINTERS_VS,
 110                      ._3DCommandSubOpcode  = binding_table_opcodes[stage],
 111                      .PointertoVSBindingTable = surfaces.offset);
 112    }
 113
 114    return VK_SUCCESS;
 115 }
 116
 117 GENX_FUNC(GEN7, GEN7) void
 118 genX(cmd_buffer_flush_descriptor_sets)(struct anv_cmd_buffer *cmd_buffer)
 119 {
 120    VkShaderStage s;
 121    VkShaderStageFlags dirty = cmd_buffer->state.descriptors_dirty &
 122                               cmd_buffer->state.pipeline->active_stages;
 123
 124    VkResult result = VK_SUCCESS;
 125    for_each_bit(s, dirty) {
 126       result = flush_descriptor_set(cmd_buffer, s);
 127       if (result != VK_SUCCESS)
 128          break;
 129    }
 130
 131    if (result != VK_SUCCESS) {
 132       assert(result == VK_ERROR_OUT_OF_DEVICE_MEMORY);
 133
 134       result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
 135       assert(result == VK_SUCCESS);
 136
 137       /* Re-emit state base addresses so we get the new surface state base
 138        * address before we start emitting binding tables etc.
 139        */
 140       anv_cmd_buffer_emit_state_base_address(cmd_buffer);
 141
 142       /* Re-emit all active binding tables */
 143       for_each_bit(s, cmd_buffer->state.pipeline->active_stages) {
 144          result = flush_descriptor_set(cmd_buffer, s);
 145
 146          /* It had better succeed this time */
 147          assert(result == VK_SUCCESS);
 148       }
 149    }
 150
 151    cmd_buffer->state.descriptors_dirty &= ~cmd_buffer->state.pipeline->active_stages;
 152 }
 153
 154 static inline int64_t
 155 clamp_int64(int64_t x, int64_t min, int64_t max)
 156 {
 157    if (x < min)
 158       return min;
 159    else if (x < max)
 160       return x;
 161    else
 162       return max;
 163 }
 164
 165 static void
 166 emit_scissor_state(struct anv_cmd_buffer *cmd_buffer,
 167                    uint32_t count, const VkRect2D *scissors)
 168 {
 169    struct anv_state scissor_state =
 170       anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, count * 32, 32);
 171
 172    for (uint32_t i = 0; i < count; i++) {
 173       const VkRect2D *s = &scissors[i];
 174
 175       /* Since xmax and ymax are inclusive, we have to have xmax < xmin or
 176        * ymax < ymin for empty clips.  In case clip x, y, width height are all
 177        * 0, the clamps below produce 0 for xmin, ymin, xmax, ymax, which isn't
 178        * what we want. Just special case empty clips and produce a canonical
 179        * empty clip. */
 180       static const struct GEN7_SCISSOR_RECT empty_scissor = {
 181          .ScissorRectangleYMin = 1,
 182          .ScissorRectangleXMin = 1,
 183          .ScissorRectangleYMax = 0,
 184          .ScissorRectangleXMax = 0
 185       };
 186
 187       const int max = 0xffff;
 188       struct GEN7_SCISSOR_RECT scissor = {
 189          /* Do this math using int64_t so overflow gets clamped correctly. */
 190          .ScissorRectangleYMin = clamp_int64(s->offset.y, 0, max),
 191          .ScissorRectangleXMin = clamp_int64(s->offset.x, 0, max),
 192          .ScissorRectangleYMax = clamp_int64((uint64_t) s->offset.y + s->extent.height - 1, 0, max),
 193          .ScissorRectangleXMax = clamp_int64((uint64_t) s->offset.x + s->extent.width - 1, 0, max)
 194       };
 195
 196       if (s->extent.width <= 0 || s->extent.height <= 0) {
 197          GEN7_SCISSOR_RECT_pack(NULL, scissor_state.map + i * 32,
 198                                 &empty_scissor);
 199       } else {
 200          GEN7_SCISSOR_RECT_pack(NULL, scissor_state.map + i * 32, &scissor);
 201       }
 202    }
 203
 204    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_SCISSOR_STATE_POINTERS,
 205                   .ScissorRectPointer = scissor_state.offset);
 206 }
 207
 208 GENX_FUNC(GEN7, GEN7) void
 209 genX(cmd_buffer_emit_scissor)(struct anv_cmd_buffer *cmd_buffer)
 210 {
 211    if (cmd_buffer->state.dynamic.scissor.count > 0) {
 212       emit_scissor_state(cmd_buffer, cmd_buffer->state.dynamic.scissor.count,
 213                          cmd_buffer->state.dynamic.scissor.scissors);
 214    } else {
 215       /* Emit a default scissor based on the currently bound framebuffer */
 216       emit_scissor_state(cmd_buffer, 1,
 217                          &(VkRect2D) {
 218                             .offset = { .x = 0, .y = 0, },
 219                             .extent = {
 220                                .width = cmd_buffer->state.framebuffer->width,
 221                                .height = cmd_buffer->state.framebuffer->height,
 222                             },
 223                          });
 224    }
 225 }
 226
 227 static const uint32_t vk_to_gen_index_type[] = {
 228    [VK_INDEX_TYPE_UINT16]                       = INDEX_WORD,
 229    [VK_INDEX_TYPE_UINT32]                       = INDEX_DWORD,
 230 };
 231
 232 static const uint32_t restart_index_for_type[] = {
 233    [VK_INDEX_TYPE_UINT16]                    = UINT16_MAX,
 234    [VK_INDEX_TYPE_UINT32]                    = UINT32_MAX,
 235 };
 236
 237 void genX(CmdBindIndexBuffer)(
 238     VkCommandBuffer                             commandBuffer,
 239     VkBuffer                                    _buffer,
 240     VkDeviceSize                                offset,
 241     VkIndexType                                 indexType)
 242 {
 243    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 244    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 245
 246    cmd_buffer->state.dirty |= ANV_CMD_DIRTY_INDEX_BUFFER;
 247    if (ANV_IS_HASWELL)
 248       cmd_buffer->state.restart_index = restart_index_for_type[indexType];
 249    cmd_buffer->state.gen7.index_buffer = buffer;
 250    cmd_buffer->state.gen7.index_type = vk_to_gen_index_type[indexType];
 251    cmd_buffer->state.gen7.index_offset = offset;
 252 }
 253
 254 static VkResult
 255 flush_compute_descriptor_set(struct anv_cmd_buffer *cmd_buffer)
 256 {
 257    struct anv_device *device = cmd_buffer->device;
 258    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 259    struct anv_state surfaces = { 0, }, samplers = { 0, };
 260    VkResult result;
 261
 262    result = anv_cmd_buffer_emit_samplers(cmd_buffer,
 263                                          VK_SHADER_STAGE_COMPUTE, &samplers);
 264    if (result != VK_SUCCESS)
 265       return result;
 266    result = anv_cmd_buffer_emit_binding_table(cmd_buffer,
 267                                                VK_SHADER_STAGE_COMPUTE, &surfaces);
 268    if (result != VK_SUCCESS)
 269       return result;
 270
 271    struct GEN7_INTERFACE_DESCRIPTOR_DATA desc = {
 272       .KernelStartPointer = pipeline->cs_simd,
 273       .BindingTablePointer = surfaces.offset,
 274       .SamplerStatePointer = samplers.offset,
 275       .NumberofThreadsinGPGPUThreadGroup = 0 /* FIXME: Really? */
 276    };
 277
 278    uint32_t size = GEN7_INTERFACE_DESCRIPTOR_DATA_length * sizeof(uint32_t);
 279    struct anv_state state =
 280       anv_state_pool_alloc(&device->dynamic_state_pool, size, 64);
 281
 282    GEN7_INTERFACE_DESCRIPTOR_DATA_pack(NULL, state.map, &desc);
 283
 284    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_INTERFACE_DESCRIPTOR_LOAD,
 285                   .InterfaceDescriptorTotalLength = size,
 286                   .InterfaceDescriptorDataStartAddress = state.offset);
 287
 288    return VK_SUCCESS;
 289 }
 290
 291 static void
 292 cmd_buffer_flush_compute_state(struct anv_cmd_buffer *cmd_buffer)
 293 {
 294    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 295    VkResult result;
 296
 297    assert(pipeline->active_stages == VK_SHADER_STAGE_COMPUTE_BIT);
 298
 299    if (cmd_buffer->state.current_pipeline != GPGPU) {
 300       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPELINE_SELECT,
 301                      .PipelineSelection = GPGPU);
 302       cmd_buffer->state.current_pipeline = GPGPU;
 303    }
 304
 305    if (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)
 306       anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
 307
 308    if ((cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_COMPUTE_BIT) ||
 309        (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)) {
 310       /* FIXME: figure out descriptors for gen7 */
 311       result = flush_compute_descriptor_set(cmd_buffer);
 312       assert(result == VK_SUCCESS);
 313       cmd_buffer->state.descriptors_dirty &= ~VK_SHADER_STAGE_COMPUTE;
 314    }
 315
 316    cmd_buffer->state.compute_dirty = 0;
 317 }
 318
 319 static void
 320 cmd_buffer_flush_state(struct anv_cmd_buffer *cmd_buffer)
 321 {
 322    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 323    uint32_t *p;
 324
 325    uint32_t vb_emit = cmd_buffer->state.vb_dirty & pipeline->vb_used;
 326
 327    assert((pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT) == 0);
 328
 329    if (cmd_buffer->state.current_pipeline != _3D) {
 330       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPELINE_SELECT,
 331                      .PipelineSelection = _3D);
 332       cmd_buffer->state.current_pipeline = _3D;
 333    }
 334
 335    if (vb_emit) {
 336       const uint32_t num_buffers = __builtin_popcount(vb_emit);
 337       const uint32_t num_dwords = 1 + num_buffers * 4;
 338
 339       p = anv_batch_emitn(&cmd_buffer->batch, num_dwords,
 340                           GEN7_3DSTATE_VERTEX_BUFFERS);
 341       uint32_t vb, i = 0;
 342       for_each_bit(vb, vb_emit) {
 343          struct anv_buffer *buffer = cmd_buffer->state.vertex_bindings[vb].buffer;
 344          uint32_t offset = cmd_buffer->state.vertex_bindings[vb].offset;
 345
 346          struct GEN7_VERTEX_BUFFER_STATE state = {
 347             .VertexBufferIndex = vb,
 348             .BufferAccessType = pipeline->instancing_enable[vb] ? INSTANCEDATA : VERTEXDATA,
 349             .VertexBufferMemoryObjectControlState = GEN7_MOCS,
 350             .AddressModifyEnable = true,
 351             .BufferPitch = pipeline->binding_stride[vb],
 352             .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
 353             .EndAddress = { buffer->bo, buffer->offset + buffer->size - 1},
 354             .InstanceDataStepRate = 1
 355          };
 356
 357          GEN7_VERTEX_BUFFER_STATE_pack(&cmd_buffer->batch, &p[1 + i * 4], &state);
 358          i++;
 359       }
 360    }
 361
 362    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_PIPELINE) {
 363       /* If somebody compiled a pipeline after starting a command buffer the
 364        * scratch bo may have grown since we started this cmd buffer (and
 365        * emitted STATE_BASE_ADDRESS).  If we're binding that pipeline now,
 366        * reemit STATE_BASE_ADDRESS so that we use the bigger scratch bo. */
 367       if (cmd_buffer->state.scratch_size < pipeline->total_scratch)
 368          gen7_cmd_buffer_emit_state_base_address(cmd_buffer);
 369
 370       anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
 371    }
 372
 373    if (cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_VERTEX_BIT ||
 374        cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_VERTEX_BIT) {
 375       /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
 376        *
 377        *    "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
 378        *    stall needs to be sent just prior to any 3DSTATE_VS,
 379        *    3DSTATE_URB_VS, 3DSTATE_CONSTANT_VS,
 380        *    3DSTATE_BINDING_TABLE_POINTER_VS,
 381        *    3DSTATE_SAMPLER_STATE_POINTER_VS command.  Only one
 382        *    PIPE_CONTROL needs to be sent before any combination of VS
 383        *    associated 3DSTATE."
 384        */
 385       anv_batch_emit(&cmd_buffer->batch, GEN7_PIPE_CONTROL,
 386                      .DepthStallEnable = true,
 387                      .PostSyncOperation = WriteImmediateData,
 388                      .Address = { &cmd_buffer->device->workaround_bo, 0 });
 389    }
 390
 391    if (cmd_buffer->state.descriptors_dirty)
 392       gen7_cmd_buffer_flush_descriptor_sets(cmd_buffer);
 393
 394    if (cmd_buffer->state.push_constants_dirty)
 395       cmd_buffer_flush_push_constants(cmd_buffer);
 396
 397    /* We use the gen8 state here because it only contains the additional
 398     * min/max fields and, since they occur at the end of the packet and
 399     * don't change the stride, they work on gen7 too.
 400     */
 401    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_VIEWPORT)
 402       gen8_cmd_buffer_emit_viewport(cmd_buffer);
 403
 404    if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_SCISSOR)
 405       gen7_cmd_buffer_emit_scissor(cmd_buffer);
 406
 407    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 408                                   ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH |
 409                                   ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS)) {
 410
 411       bool enable_bias = cmd_buffer->state.dynamic.depth_bias.bias != 0.0f ||
 412          cmd_buffer->state.dynamic.depth_bias.slope != 0.0f;
 413
 414       uint32_t sf_dw[GEN7_3DSTATE_SF_length];
 415       struct GEN7_3DSTATE_SF sf = {
 416          GEN7_3DSTATE_SF_header,
 417          .LineWidth = cmd_buffer->state.dynamic.line_width,
 418          .GlobalDepthOffsetEnableSolid = enable_bias,
 419          .GlobalDepthOffsetEnableWireframe = enable_bias,
 420          .GlobalDepthOffsetEnablePoint = enable_bias,
 421          .GlobalDepthOffsetConstant = cmd_buffer->state.dynamic.depth_bias.bias,
 422          .GlobalDepthOffsetScale = cmd_buffer->state.dynamic.depth_bias.slope,
 423          .GlobalDepthOffsetClamp = cmd_buffer->state.dynamic.depth_bias.clamp
 424       };
 425       GEN7_3DSTATE_SF_pack(NULL, sf_dw, &sf);
 426
 427       anv_batch_emit_merge(&cmd_buffer->batch, sf_dw, pipeline->gen7.sf);
 428    }
 429
 430    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS |
 431                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE)) {
 432       struct anv_state cc_state =
 433          anv_cmd_buffer_alloc_dynamic_state(cmd_buffer,
 434                                             GEN7_COLOR_CALC_STATE_length, 64);
 435       struct GEN7_COLOR_CALC_STATE cc = {
 436          .BlendConstantColorRed = cmd_buffer->state.dynamic.blend_constants[0],
 437          .BlendConstantColorGreen = cmd_buffer->state.dynamic.blend_constants[1],
 438          .BlendConstantColorBlue = cmd_buffer->state.dynamic.blend_constants[2],
 439          .BlendConstantColorAlpha = cmd_buffer->state.dynamic.blend_constants[3],
 440          .StencilReferenceValue =
 441             cmd_buffer->state.dynamic.stencil_reference.front,
 442          .BackFaceStencilReferenceValue =
 443             cmd_buffer->state.dynamic.stencil_reference.back,
 444       };
 445       GEN7_COLOR_CALC_STATE_pack(NULL, cc_state.map, &cc);
 446
 447       anv_batch_emit(&cmd_buffer->batch,
 448                      GEN7_3DSTATE_CC_STATE_POINTERS,
 449                      .ColorCalcStatePointer = cc_state.offset);
 450    }
 451
 452    if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 453                                   ANV_CMD_DIRTY_RENDER_TARGETS |
 454                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK |
 455                                   ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK)) {
 456       uint32_t depth_stencil_dw[GEN7_DEPTH_STENCIL_STATE_length];
 457
 458       const struct anv_image_view *iview =
 459          anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
 460
 461       struct GEN7_DEPTH_STENCIL_STATE depth_stencil = {
 462          .StencilBufferWriteEnable = iview && iview->format->has_stencil,
 463
 464          .StencilTestMask =
 465             cmd_buffer->state.dynamic.stencil_compare_mask.front & 0xff,
 466          .StencilWriteMask =
 467             cmd_buffer->state.dynamic.stencil_write_mask.front & 0xff,
 468
 469          .BackfaceStencilTestMask =
 470             cmd_buffer->state.dynamic.stencil_compare_mask.back & 0xff,
 471          .BackfaceStencilWriteMask =
 472             cmd_buffer->state.dynamic.stencil_write_mask.back & 0xff,
 473       };
 474       GEN7_DEPTH_STENCIL_STATE_pack(NULL, depth_stencil_dw, &depth_stencil);
 475
 476       struct anv_state ds_state =
 477          anv_cmd_buffer_merge_dynamic(cmd_buffer, depth_stencil_dw,
 478                                       pipeline->gen7.depth_stencil_state,
 479                                       GEN7_DEPTH_STENCIL_STATE_length, 64);
 480
 481       anv_batch_emit(&cmd_buffer->batch,
 482                      GEN7_3DSTATE_DEPTH_STENCIL_STATE_POINTERS,
 483                      .PointertoDEPTH_STENCIL_STATE = ds_state.offset);
 484    }
 485
 486    if (cmd_buffer->state.gen7.index_buffer &&
 487        cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
 488                                   ANV_CMD_DIRTY_INDEX_BUFFER)) {
 489       struct anv_buffer *buffer = cmd_buffer->state.gen7.index_buffer;
 490       uint32_t offset = cmd_buffer->state.gen7.index_offset;
 491
 492       if (ANV_IS_HASWELL) {
 493          anv_batch_emit(&cmd_buffer->batch, GEN75_3DSTATE_VF,
 494                         .IndexedDrawCutIndexEnable = pipeline->primitive_restart,
 495                         .CutIndex = cmd_buffer->state.restart_index);
 496       }
 497
 498       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_INDEX_BUFFER,
 499                      .CutIndexEnable = pipeline->primitive_restart,
 500                      .IndexFormat = cmd_buffer->state.gen7.index_type,
 501                      .MemoryObjectControlState = GEN7_MOCS,
 502                      .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
 503                      .BufferEndingAddress = { buffer->bo, buffer->offset + buffer->size });
 504    }
 505
 506    cmd_buffer->state.vb_dirty &= ~vb_emit;
 507    cmd_buffer->state.dirty = 0;
 508 }
 509
 510 void genX(CmdDraw)(
 511     VkCommandBuffer                             commandBuffer,
 512     uint32_t                                    vertexCount,
 513     uint32_t                                    instanceCount,
 514     uint32_t                                    firstVertex,
 515     uint32_t                                    firstInstance)
 516 {
 517    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 518    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 519
 520    cmd_buffer_flush_state(cmd_buffer);
 521
 522    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 523                   .VertexAccessType = SEQUENTIAL,
 524                   .PrimitiveTopologyType = pipeline->topology,
 525                   .VertexCountPerInstance = vertexCount,
 526                   .StartVertexLocation = firstVertex,
 527                   .InstanceCount = instanceCount,
 528                   .StartInstanceLocation = firstInstance,
 529                   .BaseVertexLocation = 0);
 530 }
 531
 532 void genX(CmdDrawIndexed)(
 533     VkCommandBuffer                             commandBuffer,
 534     uint32_t                                    indexCount,
 535     uint32_t                                    instanceCount,
 536     uint32_t                                    firstIndex,
 537     int32_t                                     vertexOffset,
 538     uint32_t                                    firstInstance)
 539 {
 540    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 541    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 542
 543    cmd_buffer_flush_state(cmd_buffer);
 544
 545    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 546                   .VertexAccessType = RANDOM,
 547                   .PrimitiveTopologyType = pipeline->topology,
 548                   .VertexCountPerInstance = indexCount,
 549                   .StartVertexLocation = firstIndex,
 550                   .InstanceCount = instanceCount,
 551                   .StartInstanceLocation = firstInstance,
 552                   .BaseVertexLocation = vertexOffset);
 553 }
 554
 555 static void
 556 gen7_batch_lrm(struct anv_batch *batch,
 557               uint32_t reg, struct anv_bo *bo, uint32_t offset)
 558 {
 559    anv_batch_emit(batch, GEN7_MI_LOAD_REGISTER_MEM,
 560                   .RegisterAddress = reg,
 561                   .MemoryAddress = { bo, offset });
 562 }
 563
 564 static void
 565 gen7_batch_lri(struct anv_batch *batch, uint32_t reg, uint32_t imm)
 566 {
 567    anv_batch_emit(batch, GEN7_MI_LOAD_REGISTER_IMM,
 568                   .RegisterOffset = reg,
 569                   .DataDWord = imm);
 570 }
 571
 572 /* Auto-Draw / Indirect Registers */
 573 #define GEN7_3DPRIM_END_OFFSET          0x2420
 574 #define GEN7_3DPRIM_START_VERTEX        0x2430
 575 #define GEN7_3DPRIM_VERTEX_COUNT        0x2434
 576 #define GEN7_3DPRIM_INSTANCE_COUNT      0x2438
 577 #define GEN7_3DPRIM_START_INSTANCE      0x243C
 578 #define GEN7_3DPRIM_BASE_VERTEX         0x2440
 579
 580 void genX(CmdDrawIndirect)(
 581     VkCommandBuffer                             commandBuffer,
 582     VkBuffer                                    _buffer,
 583     VkDeviceSize                                offset,
 584     uint32_t                                    drawCount,
 585     uint32_t                                    stride)
 586 {
 587    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 588    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 589    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 590    struct anv_bo *bo = buffer->bo;
 591    uint32_t bo_offset = buffer->offset + offset;
 592
 593    cmd_buffer_flush_state(cmd_buffer);
 594
 595    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
 596    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
 597    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
 598    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 12);
 599    gen7_batch_lri(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, 0);
 600
 601    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 602                   .IndirectParameterEnable = true,
 603                   .VertexAccessType = SEQUENTIAL,
 604                   .PrimitiveTopologyType = pipeline->topology);
 605 }
 606
 607 void genX(CmdDrawIndexedIndirect)(
 608     VkCommandBuffer                             commandBuffer,
 609     VkBuffer                                    _buffer,
 610     VkDeviceSize                                offset,
 611     uint32_t                                    drawCount,
 612     uint32_t                                    stride)
 613 {
 614    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 615    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 616    struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
 617    struct anv_bo *bo = buffer->bo;
 618    uint32_t bo_offset = buffer->offset + offset;
 619
 620    cmd_buffer_flush_state(cmd_buffer);
 621
 622    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
 623    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
 624    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
 625    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, bo, bo_offset + 12);
 626    gen7_batch_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 16);
 627
 628    anv_batch_emit(&cmd_buffer->batch, GEN7_3DPRIMITIVE,
 629                   .IndirectParameterEnable = true,
 630                   .VertexAccessType = RANDOM,
 631                   .PrimitiveTopologyType = pipeline->topology);
 632 }
 633
 634 void genX(CmdDispatch)(
 635     VkCommandBuffer                             commandBuffer,
 636     uint32_t                                    x,
 637     uint32_t                                    y,
 638     uint32_t                                    z)
 639 {
 640    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 641    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 642    struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
 643
 644    cmd_buffer_flush_compute_state(cmd_buffer);
 645
 646    anv_batch_emit(&cmd_buffer->batch, GEN7_GPGPU_WALKER,
 647                   .SIMDSize = prog_data->simd_size / 16,
 648                   .ThreadDepthCounterMaximum = 0,
 649                   .ThreadHeightCounterMaximum = 0,
 650                   .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max,
 651                   .ThreadGroupIDXDimension = x,
 652                   .ThreadGroupIDYDimension = y,
 653                   .ThreadGroupIDZDimension = z,
 654                   .RightExecutionMask = pipeline->cs_right_mask,
 655                   .BottomExecutionMask = 0xffffffff);
 656
 657    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_STATE_FLUSH);
 658 }
 659
 660 #define GPGPU_DISPATCHDIMX 0x2500
 661 #define GPGPU_DISPATCHDIMY 0x2504
 662 #define GPGPU_DISPATCHDIMZ 0x2508
 663
 664 void genX(CmdDispatchIndirect)(
 665     VkCommandBuffer                             commandBuffer,
 666     VkBuffer                                    _buffer,
 667     VkDeviceSize                                offset)
 668 {
 669    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 670    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
 671    struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
 672    struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
 673    struct anv_bo *bo = buffer->bo;
 674    uint32_t bo_offset = buffer->offset + offset;
 675
 676    cmd_buffer_flush_compute_state(cmd_buffer);
 677
 678    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMX, bo, bo_offset);
 679    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMY, bo, bo_offset + 4);
 680    gen7_batch_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMZ, bo, bo_offset + 8);
 681
 682    anv_batch_emit(&cmd_buffer->batch, GEN7_GPGPU_WALKER,
 683                   .IndirectParameterEnable = true,
 684                   .SIMDSize = prog_data->simd_size / 16,
 685                   .ThreadDepthCounterMaximum = 0,
 686                   .ThreadHeightCounterMaximum = 0,
 687                   .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max,
 688                   .RightExecutionMask = pipeline->cs_right_mask,
 689                   .BottomExecutionMask = 0xffffffff);
 690
 691    anv_batch_emit(&cmd_buffer->batch, GEN7_MEDIA_STATE_FLUSH);
 692 }
 693
 694 static void
 695 cmd_buffer_emit_depth_stencil(struct anv_cmd_buffer *cmd_buffer)
 696 {
 697    const struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
 698    const struct anv_image_view *iview =
 699       anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
 700    const struct anv_image *image = iview ? iview->image : NULL;
 701    const bool has_depth = iview && iview->format->depth_format;
 702    const bool has_stencil = iview && iview->format->has_stencil;
 703
 704    /* Emit 3DSTATE_DEPTH_BUFFER */
 705    if (has_depth) {
 706       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
 707          .SurfaceType = SURFTYPE_2D,
 708          .DepthWriteEnable = iview->format->depth_format,
 709          .StencilWriteEnable = has_stencil,
 710          .HierarchicalDepthBufferEnable = false,
 711          .SurfaceFormat = iview->format->depth_format,
 712          .SurfacePitch = image->depth_surface.stride - 1,
 713          .SurfaceBaseAddress = {
 714             .bo = image->bo,
 715             .offset = image->depth_surface.offset,
 716          },
 717          .Height = fb->height - 1,
 718          .Width = fb->width - 1,
 719          .LOD = 0,
 720          .Depth = 1 - 1,
 721          .MinimumArrayElement = 0,
 722          .DepthBufferObjectControlState = GENX(MOCS),
 723          .RenderTargetViewExtent = 1 - 1);
 724    } else {
 725       /* Even when no depth buffer is present, the hardware requires that
 726        * 3DSTATE_DEPTH_BUFFER be programmed correctly. The Broadwell PRM says:
 727        *
 728        *    If a null depth buffer is bound, the driver must instead bind depth as:
 729        *       3DSTATE_DEPTH.SurfaceType = SURFTYPE_2D
 730        *       3DSTATE_DEPTH.Width = 1
 731        *       3DSTATE_DEPTH.Height = 1
 732        *       3DSTATE_DEPTH.SuraceFormat = D16_UNORM
 733        *       3DSTATE_DEPTH.SurfaceBaseAddress = 0
 734        *       3DSTATE_DEPTH.HierarchicalDepthBufferEnable = 0
 735        *       3DSTATE_WM_DEPTH_STENCIL.DepthTestEnable = 0
 736        *       3DSTATE_WM_DEPTH_STENCIL.DepthBufferWriteEnable = 0
 737        *
 738        * The PRM is wrong, though. The width and height must be programmed to
 739        * actual framebuffer's width and height, even when neither depth buffer
 740        * nor stencil buffer is present.
 741        */
 742       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
 743          .SurfaceType = SURFTYPE_2D,
 744          .SurfaceFormat = D16_UNORM,
 745          .Width = fb->width - 1,
 746          .Height = fb->height - 1,
 747          .StencilWriteEnable = has_stencil);
 748    }
 749
 750    /* Emit 3DSTATE_STENCIL_BUFFER */
 751    if (has_stencil) {
 752       anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER),
 753 #     if (ANV_IS_HASWELL)
 754          .StencilBufferEnable = true,
 755 #     endif
 756          .StencilBufferObjectControlState = GENX(MOCS),
 757
 758          /* Stencil buffers have strange pitch. The PRM says:
 759           *
 760           *    The pitch must be set to 2x the value computed based on width,
 761           *    as the stencil buffer is stored with two rows interleaved.
 762           */
 763          .SurfacePitch = 2 * image->stencil_surface.stride - 1,
 764
 765          .SurfaceBaseAddress = {
 766             .bo = image->bo,
 767             .offset = image->offset + image->stencil_surface.offset,
 768          });
 769    } else {
 770       anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_STENCIL_BUFFER);
 771    }
 772
 773    /* Disable hierarchial depth buffers. */
 774    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_HIER_DEPTH_BUFFER);
 775
 776    /* Clear the clear params. */
 777    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_CLEAR_PARAMS);
 778 }
 779
 780 GENX_FUNC(GEN7, GEN7) void
 781 genX(cmd_buffer_begin_subpass)(struct anv_cmd_buffer *cmd_buffer,
 782                                struct anv_subpass *subpass)
 783 {
 784    cmd_buffer->state.subpass = subpass;
 785    cmd_buffer->state.descriptors_dirty |= VK_SHADER_STAGE_FRAGMENT_BIT;
 786    cmd_buffer->state.dirty |= ANV_CMD_DIRTY_RENDER_TARGETS;
 787
 788    cmd_buffer_emit_depth_stencil(cmd_buffer);
 789 }
 790
 791 static void
 792 begin_render_pass(struct anv_cmd_buffer *cmd_buffer,
 793                   const VkRenderPassBeginInfo* pRenderPassBegin)
 794 {
 795    ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
 796    ANV_FROM_HANDLE(anv_framebuffer, framebuffer, pRenderPassBegin->framebuffer);
 797
 798    cmd_buffer->state.framebuffer = framebuffer;
 799    cmd_buffer->state.pass = pass;
 800
 801    const VkRect2D *render_area = &pRenderPassBegin->renderArea;
 802
 803    anv_batch_emit(&cmd_buffer->batch, GEN7_3DSTATE_DRAWING_RECTANGLE,
 804                   .ClippedDrawingRectangleYMin = render_area->offset.y,
 805                   .ClippedDrawingRectangleXMin = render_area->offset.x,
 806                   .ClippedDrawingRectangleYMax =
 807                      render_area->offset.y + render_area->extent.height - 1,
 808                   .ClippedDrawingRectangleXMax =
 809                      render_area->offset.x + render_area->extent.width - 1,
 810                   .DrawingRectangleOriginY = 0,
 811                   .DrawingRectangleOriginX = 0);
 812
 813    anv_cmd_buffer_clear_attachments(cmd_buffer, pass,
 814                                     pRenderPassBegin->pClearValues);
 815 }
 816
 817 void genX(CmdBeginRenderPass)(
 818     VkCommandBuffer                             commandBuffer,
 819     const VkRenderPassBeginInfo*                pRenderPassBegin,
 820     VkSubpassContents                           contents)
 821 {
 822    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 823    ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
 824
 825    begin_render_pass(cmd_buffer, pRenderPassBegin);
 826
 827    gen7_cmd_buffer_begin_subpass(cmd_buffer, pass->subpasses);
 828 }
 829
 830 void genX(CmdNextSubpass)(
 831     VkCommandBuffer                             commandBuffer,
 832     VkSubpassContents                           contents)
 833 {
 834    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 835
 836    assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
 837
 838    gen7_cmd_buffer_begin_subpass(cmd_buffer, cmd_buffer->state.subpass + 1);
 839 }
 840
 841 void genX(CmdEndRenderPass)(
 842     VkCommandBuffer                             commandBuffer)
 843 {
 844    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
 845
 846    /* Emit a flushing pipe control at the end of a pass.  This is kind of a
 847     * hack but it ensures that render targets always actually get written.
 848     * Eventually, we should do flushing based on image format transitions
 849     * or something of that nature.
 850     */
 851    anv_batch_emit(&cmd_buffer->batch, GEN7_PIPE_CONTROL,
 852                   .PostSyncOperation = NoWrite,
 853                   .RenderTargetCacheFlushEnable = true,
 854                   .InstructionCacheInvalidateEnable = true,
 855                   .DepthCacheFlushEnable = true,
 856                   .VFCacheInvalidationEnable = true,
 857                   .TextureCacheInvalidationEnable = true,
 858                   .CommandStreamerStallEnable = true);
 859 }