4d14fcd2f733f22006fde25541e9f0e32ea79ac5
[mesa.git] / src / intel / vulkan / genX_cmd_buffer.c
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26
27 #include "anv_private.h"
28 #include "vk_format_info.h"
29
30 #include "common/gen_l3_config.h"
31 #include "genxml/gen_macros.h"
32 #include "genxml/genX_pack.h"
33
34 static void
35 emit_lrm(struct anv_batch *batch,
36 uint32_t reg, struct anv_bo *bo, uint32_t offset)
37 {
38 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
39 lrm.RegisterAddress = reg;
40 lrm.MemoryAddress = (struct anv_address) { bo, offset };
41 }
42 }
43
44 static void
45 emit_lri(struct anv_batch *batch, uint32_t reg, uint32_t imm)
46 {
47 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
48 lri.RegisterOffset = reg;
49 lri.DataDWord = imm;
50 }
51 }
52
53 void
54 genX(cmd_buffer_emit_state_base_address)(struct anv_cmd_buffer *cmd_buffer)
55 {
56 struct anv_device *device = cmd_buffer->device;
57
58 /* Emit a render target cache flush.
59 *
60 * This isn't documented anywhere in the PRM. However, it seems to be
61 * necessary prior to changing the surface state base adress. Without
62 * this, we get GPU hangs when using multi-level command buffers which
63 * clear depth, reset state base address, and then go render stuff.
64 */
65 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
66 pc.DCFlushEnable = true;
67 pc.RenderTargetCacheFlushEnable = true;
68 pc.CommandStreamerStallEnable = true;
69 }
70
71 anv_batch_emit(&cmd_buffer->batch, GENX(STATE_BASE_ADDRESS), sba) {
72 sba.GeneralStateBaseAddress = (struct anv_address) { NULL, 0 };
73 sba.GeneralStateMemoryObjectControlState = GENX(MOCS);
74 sba.GeneralStateBaseAddressModifyEnable = true;
75
76 sba.SurfaceStateBaseAddress =
77 anv_cmd_buffer_surface_base_address(cmd_buffer);
78 sba.SurfaceStateMemoryObjectControlState = GENX(MOCS);
79 sba.SurfaceStateBaseAddressModifyEnable = true;
80
81 sba.DynamicStateBaseAddress =
82 (struct anv_address) { &device->dynamic_state_block_pool.bo, 0 };
83 sba.DynamicStateMemoryObjectControlState = GENX(MOCS);
84 sba.DynamicStateBaseAddressModifyEnable = true;
85
86 sba.IndirectObjectBaseAddress = (struct anv_address) { NULL, 0 };
87 sba.IndirectObjectMemoryObjectControlState = GENX(MOCS);
88 sba.IndirectObjectBaseAddressModifyEnable = true;
89
90 sba.InstructionBaseAddress =
91 (struct anv_address) { &device->instruction_block_pool.bo, 0 };
92 sba.InstructionMemoryObjectControlState = GENX(MOCS);
93 sba.InstructionBaseAddressModifyEnable = true;
94
95 # if (GEN_GEN >= 8)
96 /* Broadwell requires that we specify a buffer size for a bunch of
97 * these fields. However, since we will be growing the BO's live, we
98 * just set them all to the maximum.
99 */
100 sba.GeneralStateBufferSize = 0xfffff;
101 sba.GeneralStateBufferSizeModifyEnable = true;
102 sba.DynamicStateBufferSize = 0xfffff;
103 sba.DynamicStateBufferSizeModifyEnable = true;
104 sba.IndirectObjectBufferSize = 0xfffff;
105 sba.IndirectObjectBufferSizeModifyEnable = true;
106 sba.InstructionBufferSize = 0xfffff;
107 sba.InstructionBuffersizeModifyEnable = true;
108 # endif
109 }
110
111 /* After re-setting the surface state base address, we have to do some
112 * cache flusing so that the sampler engine will pick up the new
113 * SURFACE_STATE objects and binding tables. From the Broadwell PRM,
114 * Shared Function > 3D Sampler > State > State Caching (page 96):
115 *
116 * Coherency with system memory in the state cache, like the texture
117 * cache is handled partially by software. It is expected that the
118 * command stream or shader will issue Cache Flush operation or
119 * Cache_Flush sampler message to ensure that the L1 cache remains
120 * coherent with system memory.
121 *
122 * [...]
123 *
124 * Whenever the value of the Dynamic_State_Base_Addr,
125 * Surface_State_Base_Addr are altered, the L1 state cache must be
126 * invalidated to ensure the new surface or sampler state is fetched
127 * from system memory.
128 *
129 * The PIPE_CONTROL command has a "State Cache Invalidation Enable" bit
130 * which, according the PIPE_CONTROL instruction documentation in the
131 * Broadwell PRM:
132 *
133 * Setting this bit is independent of any other bit in this packet.
134 * This bit controls the invalidation of the L1 and L2 state caches
135 * at the top of the pipe i.e. at the parsing time.
136 *
137 * Unfortunately, experimentation seems to indicate that state cache
138 * invalidation through a PIPE_CONTROL does nothing whatsoever in
139 * regards to surface state and binding tables. In stead, it seems that
140 * invalidating the texture cache is what is actually needed.
141 *
142 * XXX: As far as we have been able to determine through
143 * experimentation, shows that flush the texture cache appears to be
144 * sufficient. The theory here is that all of the sampling/rendering
145 * units cache the binding table in the texture cache. However, we have
146 * yet to be able to actually confirm this.
147 */
148 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
149 pc.TextureCacheInvalidationEnable = true;
150 pc.ConstantCacheInvalidationEnable = true;
151 pc.StateCacheInvalidationEnable = true;
152 }
153 }
154
155 static void
156 add_surface_state_reloc(struct anv_cmd_buffer *cmd_buffer,
157 struct anv_state state,
158 struct anv_bo *bo, uint32_t offset)
159 {
160 const struct isl_device *isl_dev = &cmd_buffer->device->isl_dev;
161
162 VkResult result =
163 anv_reloc_list_add(&cmd_buffer->surface_relocs, &cmd_buffer->pool->alloc,
164 state.offset + isl_dev->ss.addr_offset, bo, offset);
165 if (result != VK_SUCCESS)
166 anv_batch_set_error(&cmd_buffer->batch, result);
167 }
168
169 static void
170 add_image_view_relocs(struct anv_cmd_buffer *cmd_buffer,
171 const struct anv_image_view *iview,
172 enum isl_aux_usage aux_usage,
173 struct anv_state state)
174 {
175 const struct isl_device *isl_dev = &cmd_buffer->device->isl_dev;
176
177 add_surface_state_reloc(cmd_buffer, state, iview->bo, iview->offset);
178
179 if (aux_usage != ISL_AUX_USAGE_NONE) {
180 uint32_t aux_offset = iview->offset + iview->image->aux_surface.offset;
181
182 /* On gen7 and prior, the bottom 12 bits of the MCS base address are
183 * used to store other information. This should be ok, however, because
184 * surface buffer addresses are always 4K page alinged.
185 */
186 assert((aux_offset & 0xfff) == 0);
187 uint32_t *aux_addr_dw = state.map + isl_dev->ss.aux_addr_offset;
188 aux_offset += *aux_addr_dw & 0xfff;
189
190 VkResult result =
191 anv_reloc_list_add(&cmd_buffer->surface_relocs,
192 &cmd_buffer->pool->alloc,
193 state.offset + isl_dev->ss.aux_addr_offset,
194 iview->bo, aux_offset);
195 if (result != VK_SUCCESS)
196 anv_batch_set_error(&cmd_buffer->batch, result);
197 }
198 }
199
200 static bool
201 color_is_zero_one(VkClearColorValue value, enum isl_format format)
202 {
203 if (isl_format_has_int_channel(format)) {
204 for (unsigned i = 0; i < 4; i++) {
205 if (value.int32[i] != 0 && value.int32[i] != 1)
206 return false;
207 }
208 } else {
209 for (unsigned i = 0; i < 4; i++) {
210 if (value.float32[i] != 0.0f && value.float32[i] != 1.0f)
211 return false;
212 }
213 }
214
215 return true;
216 }
217
218 static void
219 color_attachment_compute_aux_usage(struct anv_device *device,
220 struct anv_attachment_state *att_state,
221 struct anv_image_view *iview,
222 VkRect2D render_area,
223 union isl_color_value *fast_clear_color)
224 {
225 if (iview->image->aux_surface.isl.size == 0) {
226 att_state->aux_usage = ISL_AUX_USAGE_NONE;
227 att_state->input_aux_usage = ISL_AUX_USAGE_NONE;
228 att_state->fast_clear = false;
229 return;
230 } else if (iview->image->aux_usage == ISL_AUX_USAGE_MCS) {
231 att_state->aux_usage = ISL_AUX_USAGE_MCS;
232 att_state->input_aux_usage = ISL_AUX_USAGE_MCS;
233 att_state->fast_clear = false;
234 return;
235 }
236
237 assert(iview->image->aux_surface.isl.usage & ISL_SURF_USAGE_CCS_BIT);
238
239 att_state->clear_color_is_zero_one =
240 color_is_zero_one(att_state->clear_value.color, iview->isl.format);
241
242 if (att_state->pending_clear_aspects == VK_IMAGE_ASPECT_COLOR_BIT) {
243 /* Start off assuming fast clears are possible */
244 att_state->fast_clear = true;
245
246 /* Potentially, we could do partial fast-clears but doing so has crazy
247 * alignment restrictions. It's easier to just restrict to full size
248 * fast clears for now.
249 */
250 if (render_area.offset.x != 0 ||
251 render_area.offset.y != 0 ||
252 render_area.extent.width != iview->extent.width ||
253 render_area.extent.height != iview->extent.height)
254 att_state->fast_clear = false;
255
256 if (GEN_GEN <= 7) {
257 /* On gen7, we can't do multi-LOD or multi-layer fast-clears. We
258 * technically can, but it comes with crazy restrictions that we
259 * don't want to deal with now.
260 */
261 if (iview->isl.base_level > 0 ||
262 iview->isl.base_array_layer > 0 ||
263 iview->isl.array_len > 1)
264 att_state->fast_clear = false;
265 }
266
267 /* On Broadwell and earlier, we can only handle 0/1 clear colors */
268 if (GEN_GEN <= 8 && !att_state->clear_color_is_zero_one)
269 att_state->fast_clear = false;
270
271 if (att_state->fast_clear) {
272 memcpy(fast_clear_color->u32, att_state->clear_value.color.uint32,
273 sizeof(fast_clear_color->u32));
274 }
275 } else {
276 att_state->fast_clear = false;
277 }
278
279 /**
280 * TODO: Consider using a heuristic to determine if temporarily enabling
281 * CCS_E for this image view would be beneficial.
282 *
283 * While fast-clear resolves and partial resolves are fairly cheap in the
284 * case where you render to most of the pixels, full resolves are not
285 * because they potentially involve reading and writing the entire
286 * framebuffer. If we can't texture with CCS_E, we should leave it off and
287 * limit ourselves to fast clears.
288 */
289 if (iview->image->aux_usage == ISL_AUX_USAGE_CCS_E) {
290 att_state->aux_usage = ISL_AUX_USAGE_CCS_E;
291 att_state->input_aux_usage = ISL_AUX_USAGE_CCS_E;
292 } else if (att_state->fast_clear) {
293 att_state->aux_usage = ISL_AUX_USAGE_CCS_D;
294 if (GEN_GEN >= 9 &&
295 !isl_format_supports_ccs_e(&device->info, iview->isl.format)) {
296 /* From the Sky Lake PRM, RENDER_SURFACE_STATE::AuxiliarySurfaceMode:
297 *
298 * "If Number of Multisamples is MULTISAMPLECOUNT_1, AUX_CCS_D
299 * setting is only allowed if Surface Format supported for Fast
300 * Clear. In addition, if the surface is bound to the sampling
301 * engine, Surface Format must be supported for Render Target
302 * Compression for surfaces bound to the sampling engine."
303 *
304 * In other words, we can't sample from a fast-cleared image if it
305 * doesn't also support color compression.
306 */
307 att_state->input_aux_usage = ISL_AUX_USAGE_NONE;
308 } else if (GEN_GEN == 8) {
309 /* Broadwell can sample from fast-cleared images */
310 att_state->input_aux_usage = ISL_AUX_USAGE_CCS_D;
311 } else {
312 /* Ivy Bridge and Haswell cannot */
313 att_state->input_aux_usage = ISL_AUX_USAGE_NONE;
314 }
315 } else {
316 att_state->aux_usage = ISL_AUX_USAGE_NONE;
317 att_state->input_aux_usage = ISL_AUX_USAGE_NONE;
318 }
319 }
320
321 static bool
322 need_input_attachment_state(const struct anv_render_pass_attachment *att)
323 {
324 if (!(att->usage & VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT))
325 return false;
326
327 /* We only allocate input attachment states for color surfaces. Compression
328 * is not yet enabled for depth textures and stencil doesn't allow
329 * compression so we can just use the texture surface state from the view.
330 */
331 return vk_format_is_color(att->format);
332 }
333
334 /* Transitions a HiZ-enabled depth buffer from one layout to another. Unless
335 * the initial layout is undefined, the HiZ buffer and depth buffer will
336 * represent the same data at the end of this operation.
337 */
338 static void
339 transition_depth_buffer(struct anv_cmd_buffer *cmd_buffer,
340 const struct anv_image *image,
341 VkImageLayout initial_layout,
342 VkImageLayout final_layout)
343 {
344 assert(image);
345
346 /* A transition is a no-op if HiZ is not enabled, or if the initial and
347 * final layouts are equal.
348 *
349 * The undefined layout indicates that the user doesn't care about the data
350 * that's currently in the buffer. Therefore, a data-preserving resolve
351 * operation is not needed.
352 *
353 * The pre-initialized layout is equivalent to the undefined layout for
354 * optimally-tiled images. Anv only exposes support for optimally-tiled
355 * depth buffers.
356 */
357 if (image->aux_usage != ISL_AUX_USAGE_HIZ ||
358 initial_layout == final_layout ||
359 initial_layout == VK_IMAGE_LAYOUT_UNDEFINED ||
360 initial_layout == VK_IMAGE_LAYOUT_PREINITIALIZED)
361 return;
362
363 const bool hiz_enabled = ISL_AUX_USAGE_HIZ ==
364 anv_layout_to_aux_usage(&cmd_buffer->device->info, image, image->aspects,
365 initial_layout);
366 const bool enable_hiz = ISL_AUX_USAGE_HIZ ==
367 anv_layout_to_aux_usage(&cmd_buffer->device->info, image, image->aspects,
368 final_layout);
369
370 enum blorp_hiz_op hiz_op;
371 if (hiz_enabled && !enable_hiz) {
372 hiz_op = BLORP_HIZ_OP_DEPTH_RESOLVE;
373 } else if (!hiz_enabled && enable_hiz) {
374 hiz_op = BLORP_HIZ_OP_HIZ_RESOLVE;
375 } else {
376 assert(hiz_enabled == enable_hiz);
377 /* If the same buffer will be used, no resolves are necessary. */
378 hiz_op = BLORP_HIZ_OP_NONE;
379 }
380
381 if (hiz_op != BLORP_HIZ_OP_NONE)
382 anv_gen8_hiz_op_resolve(cmd_buffer, image, hiz_op);
383 }
384
385
386 /**
387 * Setup anv_cmd_state::attachments for vkCmdBeginRenderPass.
388 */
389 static VkResult
390 genX(cmd_buffer_setup_attachments)(struct anv_cmd_buffer *cmd_buffer,
391 struct anv_render_pass *pass,
392 const VkRenderPassBeginInfo *begin)
393 {
394 const struct isl_device *isl_dev = &cmd_buffer->device->isl_dev;
395 struct anv_cmd_state *state = &cmd_buffer->state;
396
397 vk_free(&cmd_buffer->pool->alloc, state->attachments);
398
399 if (pass->attachment_count == 0) {
400 state->attachments = NULL;
401 return VK_SUCCESS;
402 }
403
404 state->attachments = vk_alloc(&cmd_buffer->pool->alloc,
405 pass->attachment_count *
406 sizeof(state->attachments[0]),
407 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
408 if (state->attachments == NULL) {
409 /* Propagate VK_ERROR_OUT_OF_HOST_MEMORY to vkEndCommandBuffer */
410 return anv_batch_set_error(&cmd_buffer->batch,
411 VK_ERROR_OUT_OF_HOST_MEMORY);
412 }
413
414 bool need_null_state = false;
415 unsigned num_states = 0;
416 for (uint32_t i = 0; i < pass->attachment_count; ++i) {
417 if (vk_format_is_color(pass->attachments[i].format)) {
418 num_states++;
419 } else {
420 /* We need a null state for any depth-stencil-only subpasses.
421 * Importantly, this includes depth/stencil clears so we create one
422 * whenever we have depth or stencil
423 */
424 need_null_state = true;
425 }
426
427 if (need_input_attachment_state(&pass->attachments[i]))
428 num_states++;
429 }
430 num_states += need_null_state;
431
432 const uint32_t ss_stride = align_u32(isl_dev->ss.size, isl_dev->ss.align);
433 state->render_pass_states =
434 anv_state_stream_alloc(&cmd_buffer->surface_state_stream,
435 num_states * ss_stride, isl_dev->ss.align);
436
437 struct anv_state next_state = state->render_pass_states;
438 next_state.alloc_size = isl_dev->ss.size;
439
440 if (need_null_state) {
441 state->null_surface_state = next_state;
442 next_state.offset += ss_stride;
443 next_state.map += ss_stride;
444 }
445
446 for (uint32_t i = 0; i < pass->attachment_count; ++i) {
447 if (vk_format_is_color(pass->attachments[i].format)) {
448 state->attachments[i].color_rt_state = next_state;
449 next_state.offset += ss_stride;
450 next_state.map += ss_stride;
451 }
452
453 if (need_input_attachment_state(&pass->attachments[i])) {
454 state->attachments[i].input_att_state = next_state;
455 next_state.offset += ss_stride;
456 next_state.map += ss_stride;
457 }
458 }
459 assert(next_state.offset == state->render_pass_states.offset +
460 state->render_pass_states.alloc_size);
461
462 if (begin) {
463 ANV_FROM_HANDLE(anv_framebuffer, framebuffer, begin->framebuffer);
464 assert(pass->attachment_count == framebuffer->attachment_count);
465
466 if (need_null_state) {
467 struct GENX(RENDER_SURFACE_STATE) null_ss = {
468 .SurfaceType = SURFTYPE_NULL,
469 .SurfaceArray = framebuffer->layers > 0,
470 .SurfaceFormat = ISL_FORMAT_R8G8B8A8_UNORM,
471 #if GEN_GEN >= 8
472 .TileMode = YMAJOR,
473 #else
474 .TiledSurface = true,
475 #endif
476 .Width = framebuffer->width - 1,
477 .Height = framebuffer->height - 1,
478 .Depth = framebuffer->layers - 1,
479 .RenderTargetViewExtent = framebuffer->layers - 1,
480 };
481 GENX(RENDER_SURFACE_STATE_pack)(NULL, state->null_surface_state.map,
482 &null_ss);
483 }
484
485 for (uint32_t i = 0; i < pass->attachment_count; ++i) {
486 struct anv_render_pass_attachment *att = &pass->attachments[i];
487 VkImageAspectFlags att_aspects = vk_format_aspects(att->format);
488 VkImageAspectFlags clear_aspects = 0;
489
490 if (att_aspects == VK_IMAGE_ASPECT_COLOR_BIT) {
491 /* color attachment */
492 if (att->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
493 clear_aspects |= VK_IMAGE_ASPECT_COLOR_BIT;
494 }
495 } else {
496 /* depthstencil attachment */
497 if ((att_aspects & VK_IMAGE_ASPECT_DEPTH_BIT) &&
498 att->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
499 clear_aspects |= VK_IMAGE_ASPECT_DEPTH_BIT;
500 }
501 if ((att_aspects & VK_IMAGE_ASPECT_STENCIL_BIT) &&
502 att->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
503 clear_aspects |= VK_IMAGE_ASPECT_STENCIL_BIT;
504 }
505 }
506
507 state->attachments[i].current_layout = att->initial_layout;
508 state->attachments[i].pending_clear_aspects = clear_aspects;
509 if (clear_aspects)
510 state->attachments[i].clear_value = begin->pClearValues[i];
511
512 struct anv_image_view *iview = framebuffer->attachments[i];
513 anv_assert(iview->vk_format == att->format);
514
515 union isl_color_value clear_color = { .u32 = { 0, } };
516 if (att_aspects == VK_IMAGE_ASPECT_COLOR_BIT) {
517 color_attachment_compute_aux_usage(cmd_buffer->device,
518 &state->attachments[i],
519 iview, begin->renderArea,
520 &clear_color);
521
522 struct isl_view view = iview->isl;
523 view.usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT;
524 view.swizzle = anv_swizzle_for_render(view.swizzle);
525 isl_surf_fill_state(isl_dev,
526 state->attachments[i].color_rt_state.map,
527 .surf = &iview->image->color_surface.isl,
528 .view = &view,
529 .aux_surf = &iview->image->aux_surface.isl,
530 .aux_usage = state->attachments[i].aux_usage,
531 .clear_color = clear_color,
532 .mocs = cmd_buffer->device->default_mocs);
533
534 add_image_view_relocs(cmd_buffer, iview,
535 state->attachments[i].aux_usage,
536 state->attachments[i].color_rt_state);
537 } else {
538 /* This field will be initialized after the first subpass
539 * transition.
540 */
541 state->attachments[i].aux_usage = ISL_AUX_USAGE_NONE;
542
543 state->attachments[i].input_aux_usage = ISL_AUX_USAGE_NONE;
544 }
545
546 if (need_input_attachment_state(&pass->attachments[i])) {
547 struct isl_view view = iview->isl;
548 view.usage |= ISL_SURF_USAGE_TEXTURE_BIT;
549 isl_surf_fill_state(isl_dev,
550 state->attachments[i].input_att_state.map,
551 .surf = &iview->image->color_surface.isl,
552 .view = &view,
553 .aux_surf = &iview->image->aux_surface.isl,
554 .aux_usage = state->attachments[i].input_aux_usage,
555 .clear_color = clear_color,
556 .mocs = cmd_buffer->device->default_mocs);
557
558 add_image_view_relocs(cmd_buffer, iview,
559 state->attachments[i].input_aux_usage,
560 state->attachments[i].input_att_state);
561 }
562 }
563
564 anv_state_flush(cmd_buffer->device, state->render_pass_states);
565 }
566
567 return VK_SUCCESS;
568 }
569
570 VkResult
571 genX(BeginCommandBuffer)(
572 VkCommandBuffer commandBuffer,
573 const VkCommandBufferBeginInfo* pBeginInfo)
574 {
575 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
576
577 /* If this is the first vkBeginCommandBuffer, we must *initialize* the
578 * command buffer's state. Otherwise, we must *reset* its state. In both
579 * cases we reset it.
580 *
581 * From the Vulkan 1.0 spec:
582 *
583 * If a command buffer is in the executable state and the command buffer
584 * was allocated from a command pool with the
585 * VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT flag set, then
586 * vkBeginCommandBuffer implicitly resets the command buffer, behaving
587 * as if vkResetCommandBuffer had been called with
588 * VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT not set. It then puts
589 * the command buffer in the recording state.
590 */
591 anv_cmd_buffer_reset(cmd_buffer);
592
593 cmd_buffer->usage_flags = pBeginInfo->flags;
594
595 assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY ||
596 !(cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT));
597
598 genX(cmd_buffer_emit_state_base_address)(cmd_buffer);
599
600 VkResult result = VK_SUCCESS;
601 if (cmd_buffer->usage_flags &
602 VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) {
603 cmd_buffer->state.pass =
604 anv_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
605 cmd_buffer->state.subpass =
606 &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
607 cmd_buffer->state.framebuffer = NULL;
608
609 result = genX(cmd_buffer_setup_attachments)(cmd_buffer,
610 cmd_buffer->state.pass, NULL);
611
612 cmd_buffer->state.dirty |= ANV_CMD_DIRTY_RENDER_TARGETS;
613 }
614
615 return result;
616 }
617
618 VkResult
619 genX(EndCommandBuffer)(
620 VkCommandBuffer commandBuffer)
621 {
622 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
623
624 if (anv_batch_has_error(&cmd_buffer->batch))
625 return cmd_buffer->batch.status;
626
627 /* We want every command buffer to start with the PMA fix in a known state,
628 * so we disable it at the end of the command buffer.
629 */
630 genX(cmd_buffer_enable_pma_fix)(cmd_buffer, false);
631
632 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
633
634 anv_cmd_buffer_end_batch_buffer(cmd_buffer);
635
636 return VK_SUCCESS;
637 }
638
639 void
640 genX(CmdExecuteCommands)(
641 VkCommandBuffer commandBuffer,
642 uint32_t commandBufferCount,
643 const VkCommandBuffer* pCmdBuffers)
644 {
645 ANV_FROM_HANDLE(anv_cmd_buffer, primary, commandBuffer);
646
647 assert(primary->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
648
649 if (anv_batch_has_error(&primary->batch))
650 return;
651
652 /* The secondary command buffers will assume that the PMA fix is disabled
653 * when they begin executing. Make sure this is true.
654 */
655 genX(cmd_buffer_enable_pma_fix)(primary, false);
656
657 for (uint32_t i = 0; i < commandBufferCount; i++) {
658 ANV_FROM_HANDLE(anv_cmd_buffer, secondary, pCmdBuffers[i]);
659
660 assert(secondary->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY);
661 assert(!anv_batch_has_error(&secondary->batch));
662
663 if (secondary->usage_flags &
664 VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) {
665 /* If we're continuing a render pass from the primary, we need to
666 * copy the surface states for the current subpass into the storage
667 * we allocated for them in BeginCommandBuffer.
668 */
669 struct anv_bo *ss_bo = &primary->device->surface_state_block_pool.bo;
670 struct anv_state src_state = primary->state.render_pass_states;
671 struct anv_state dst_state = secondary->state.render_pass_states;
672 assert(src_state.alloc_size == dst_state.alloc_size);
673
674 genX(cmd_buffer_gpu_memcpy)(primary, ss_bo, dst_state.offset,
675 ss_bo, src_state.offset,
676 src_state.alloc_size);
677 }
678
679 anv_cmd_buffer_add_secondary(primary, secondary);
680 }
681
682 /* Each of the secondary command buffers will use its own state base
683 * address. We need to re-emit state base address for the primary after
684 * all of the secondaries are done.
685 *
686 * TODO: Maybe we want to make this a dirty bit to avoid extra state base
687 * address calls?
688 */
689 genX(cmd_buffer_emit_state_base_address)(primary);
690 }
691
692 #define IVB_L3SQCREG1_SQGHPCI_DEFAULT 0x00730000
693 #define VLV_L3SQCREG1_SQGHPCI_DEFAULT 0x00d30000
694 #define HSW_L3SQCREG1_SQGHPCI_DEFAULT 0x00610000
695
696 /**
697 * Program the hardware to use the specified L3 configuration.
698 */
699 void
700 genX(cmd_buffer_config_l3)(struct anv_cmd_buffer *cmd_buffer,
701 const struct gen_l3_config *cfg)
702 {
703 assert(cfg);
704 if (cfg == cmd_buffer->state.current_l3_config)
705 return;
706
707 if (unlikely(INTEL_DEBUG & DEBUG_L3)) {
708 fprintf(stderr, "L3 config transition: ");
709 gen_dump_l3_config(cfg, stderr);
710 }
711
712 const bool has_slm = cfg->n[GEN_L3P_SLM];
713
714 /* According to the hardware docs, the L3 partitioning can only be changed
715 * while the pipeline is completely drained and the caches are flushed,
716 * which involves a first PIPE_CONTROL flush which stalls the pipeline...
717 */
718 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
719 pc.DCFlushEnable = true;
720 pc.PostSyncOperation = NoWrite;
721 pc.CommandStreamerStallEnable = true;
722 }
723
724 /* ...followed by a second pipelined PIPE_CONTROL that initiates
725 * invalidation of the relevant caches. Note that because RO invalidation
726 * happens at the top of the pipeline (i.e. right away as the PIPE_CONTROL
727 * command is processed by the CS) we cannot combine it with the previous
728 * stalling flush as the hardware documentation suggests, because that
729 * would cause the CS to stall on previous rendering *after* RO
730 * invalidation and wouldn't prevent the RO caches from being polluted by
731 * concurrent rendering before the stall completes. This intentionally
732 * doesn't implement the SKL+ hardware workaround suggesting to enable CS
733 * stall on PIPE_CONTROLs with the texture cache invalidation bit set for
734 * GPGPU workloads because the previous and subsequent PIPE_CONTROLs
735 * already guarantee that there is no concurrent GPGPU kernel execution
736 * (see SKL HSD 2132585).
737 */
738 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
739 pc.TextureCacheInvalidationEnable = true;
740 pc.ConstantCacheInvalidationEnable = true;
741 pc.InstructionCacheInvalidateEnable = true;
742 pc.StateCacheInvalidationEnable = true;
743 pc.PostSyncOperation = NoWrite;
744 }
745
746 /* Now send a third stalling flush to make sure that invalidation is
747 * complete when the L3 configuration registers are modified.
748 */
749 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
750 pc.DCFlushEnable = true;
751 pc.PostSyncOperation = NoWrite;
752 pc.CommandStreamerStallEnable = true;
753 }
754
755 #if GEN_GEN >= 8
756
757 assert(!cfg->n[GEN_L3P_IS] && !cfg->n[GEN_L3P_C] && !cfg->n[GEN_L3P_T]);
758
759 uint32_t l3cr;
760 anv_pack_struct(&l3cr, GENX(L3CNTLREG),
761 .SLMEnable = has_slm,
762 .URBAllocation = cfg->n[GEN_L3P_URB],
763 .ROAllocation = cfg->n[GEN_L3P_RO],
764 .DCAllocation = cfg->n[GEN_L3P_DC],
765 .AllAllocation = cfg->n[GEN_L3P_ALL]);
766
767 /* Set up the L3 partitioning. */
768 emit_lri(&cmd_buffer->batch, GENX(L3CNTLREG_num), l3cr);
769
770 #else
771
772 const bool has_dc = cfg->n[GEN_L3P_DC] || cfg->n[GEN_L3P_ALL];
773 const bool has_is = cfg->n[GEN_L3P_IS] || cfg->n[GEN_L3P_RO] ||
774 cfg->n[GEN_L3P_ALL];
775 const bool has_c = cfg->n[GEN_L3P_C] || cfg->n[GEN_L3P_RO] ||
776 cfg->n[GEN_L3P_ALL];
777 const bool has_t = cfg->n[GEN_L3P_T] || cfg->n[GEN_L3P_RO] ||
778 cfg->n[GEN_L3P_ALL];
779
780 assert(!cfg->n[GEN_L3P_ALL]);
781
782 /* When enabled SLM only uses a portion of the L3 on half of the banks,
783 * the matching space on the remaining banks has to be allocated to a
784 * client (URB for all validated configurations) set to the
785 * lower-bandwidth 2-bank address hashing mode.
786 */
787 const struct gen_device_info *devinfo = &cmd_buffer->device->info;
788 const bool urb_low_bw = has_slm && !devinfo->is_baytrail;
789 assert(!urb_low_bw || cfg->n[GEN_L3P_URB] == cfg->n[GEN_L3P_SLM]);
790
791 /* Minimum number of ways that can be allocated to the URB. */
792 MAYBE_UNUSED const unsigned n0_urb = devinfo->is_baytrail ? 32 : 0;
793 assert(cfg->n[GEN_L3P_URB] >= n0_urb);
794
795 uint32_t l3sqcr1, l3cr2, l3cr3;
796 anv_pack_struct(&l3sqcr1, GENX(L3SQCREG1),
797 .ConvertDC_UC = !has_dc,
798 .ConvertIS_UC = !has_is,
799 .ConvertC_UC = !has_c,
800 .ConvertT_UC = !has_t);
801 l3sqcr1 |=
802 GEN_IS_HASWELL ? HSW_L3SQCREG1_SQGHPCI_DEFAULT :
803 devinfo->is_baytrail ? VLV_L3SQCREG1_SQGHPCI_DEFAULT :
804 IVB_L3SQCREG1_SQGHPCI_DEFAULT;
805
806 anv_pack_struct(&l3cr2, GENX(L3CNTLREG2),
807 .SLMEnable = has_slm,
808 .URBLowBandwidth = urb_low_bw,
809 .URBAllocation = cfg->n[GEN_L3P_URB],
810 #if !GEN_IS_HASWELL
811 .ALLAllocation = cfg->n[GEN_L3P_ALL],
812 #endif
813 .ROAllocation = cfg->n[GEN_L3P_RO],
814 .DCAllocation = cfg->n[GEN_L3P_DC]);
815
816 anv_pack_struct(&l3cr3, GENX(L3CNTLREG3),
817 .ISAllocation = cfg->n[GEN_L3P_IS],
818 .ISLowBandwidth = 0,
819 .CAllocation = cfg->n[GEN_L3P_C],
820 .CLowBandwidth = 0,
821 .TAllocation = cfg->n[GEN_L3P_T],
822 .TLowBandwidth = 0);
823
824 /* Set up the L3 partitioning. */
825 emit_lri(&cmd_buffer->batch, GENX(L3SQCREG1_num), l3sqcr1);
826 emit_lri(&cmd_buffer->batch, GENX(L3CNTLREG2_num), l3cr2);
827 emit_lri(&cmd_buffer->batch, GENX(L3CNTLREG3_num), l3cr3);
828
829 #if GEN_IS_HASWELL
830 if (cmd_buffer->device->instance->physicalDevice.cmd_parser_version >= 4) {
831 /* Enable L3 atomics on HSW if we have a DC partition, otherwise keep
832 * them disabled to avoid crashing the system hard.
833 */
834 uint32_t scratch1, chicken3;
835 anv_pack_struct(&scratch1, GENX(SCRATCH1),
836 .L3AtomicDisable = !has_dc);
837 anv_pack_struct(&chicken3, GENX(CHICKEN3),
838 .L3AtomicDisableMask = true,
839 .L3AtomicDisable = !has_dc);
840 emit_lri(&cmd_buffer->batch, GENX(SCRATCH1_num), scratch1);
841 emit_lri(&cmd_buffer->batch, GENX(CHICKEN3_num), chicken3);
842 }
843 #endif
844
845 #endif
846
847 cmd_buffer->state.current_l3_config = cfg;
848 }
849
850 void
851 genX(cmd_buffer_apply_pipe_flushes)(struct anv_cmd_buffer *cmd_buffer)
852 {
853 enum anv_pipe_bits bits = cmd_buffer->state.pending_pipe_bits;
854
855 /* Flushes are pipelined while invalidations are handled immediately.
856 * Therefore, if we're flushing anything then we need to schedule a stall
857 * before any invalidations can happen.
858 */
859 if (bits & ANV_PIPE_FLUSH_BITS)
860 bits |= ANV_PIPE_NEEDS_CS_STALL_BIT;
861
862 /* If we're going to do an invalidate and we have a pending CS stall that
863 * has yet to be resolved, we do the CS stall now.
864 */
865 if ((bits & ANV_PIPE_INVALIDATE_BITS) &&
866 (bits & ANV_PIPE_NEEDS_CS_STALL_BIT)) {
867 bits |= ANV_PIPE_CS_STALL_BIT;
868 bits &= ~ANV_PIPE_NEEDS_CS_STALL_BIT;
869 }
870
871 if (bits & (ANV_PIPE_FLUSH_BITS | ANV_PIPE_CS_STALL_BIT)) {
872 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pipe) {
873 pipe.DepthCacheFlushEnable = bits & ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
874 pipe.DCFlushEnable = bits & ANV_PIPE_DATA_CACHE_FLUSH_BIT;
875 pipe.RenderTargetCacheFlushEnable =
876 bits & ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
877
878 pipe.DepthStallEnable = bits & ANV_PIPE_DEPTH_STALL_BIT;
879 pipe.CommandStreamerStallEnable = bits & ANV_PIPE_CS_STALL_BIT;
880 pipe.StallAtPixelScoreboard = bits & ANV_PIPE_STALL_AT_SCOREBOARD_BIT;
881
882 /*
883 * According to the Broadwell documentation, any PIPE_CONTROL with the
884 * "Command Streamer Stall" bit set must also have another bit set,
885 * with five different options:
886 *
887 * - Render Target Cache Flush
888 * - Depth Cache Flush
889 * - Stall at Pixel Scoreboard
890 * - Post-Sync Operation
891 * - Depth Stall
892 * - DC Flush Enable
893 *
894 * I chose "Stall at Pixel Scoreboard" since that's what we use in
895 * mesa and it seems to work fine. The choice is fairly arbitrary.
896 */
897 if ((bits & ANV_PIPE_CS_STALL_BIT) &&
898 !(bits & (ANV_PIPE_FLUSH_BITS | ANV_PIPE_DEPTH_STALL_BIT |
899 ANV_PIPE_STALL_AT_SCOREBOARD_BIT)))
900 pipe.StallAtPixelScoreboard = true;
901 }
902
903 bits &= ~(ANV_PIPE_FLUSH_BITS | ANV_PIPE_CS_STALL_BIT);
904 }
905
906 if (bits & ANV_PIPE_INVALIDATE_BITS) {
907 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pipe) {
908 pipe.StateCacheInvalidationEnable =
909 bits & ANV_PIPE_STATE_CACHE_INVALIDATE_BIT;
910 pipe.ConstantCacheInvalidationEnable =
911 bits & ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
912 pipe.VFCacheInvalidationEnable =
913 bits & ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
914 pipe.TextureCacheInvalidationEnable =
915 bits & ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
916 pipe.InstructionCacheInvalidateEnable =
917 bits & ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT;
918 }
919
920 bits &= ~ANV_PIPE_INVALIDATE_BITS;
921 }
922
923 cmd_buffer->state.pending_pipe_bits = bits;
924 }
925
926 void genX(CmdPipelineBarrier)(
927 VkCommandBuffer commandBuffer,
928 VkPipelineStageFlags srcStageMask,
929 VkPipelineStageFlags destStageMask,
930 VkBool32 byRegion,
931 uint32_t memoryBarrierCount,
932 const VkMemoryBarrier* pMemoryBarriers,
933 uint32_t bufferMemoryBarrierCount,
934 const VkBufferMemoryBarrier* pBufferMemoryBarriers,
935 uint32_t imageMemoryBarrierCount,
936 const VkImageMemoryBarrier* pImageMemoryBarriers)
937 {
938 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
939 uint32_t b;
940
941 /* XXX: Right now, we're really dumb and just flush whatever categories
942 * the app asks for. One of these days we may make this a bit better
943 * but right now that's all the hardware allows for in most areas.
944 */
945 VkAccessFlags src_flags = 0;
946 VkAccessFlags dst_flags = 0;
947
948 for (uint32_t i = 0; i < memoryBarrierCount; i++) {
949 src_flags |= pMemoryBarriers[i].srcAccessMask;
950 dst_flags |= pMemoryBarriers[i].dstAccessMask;
951 }
952
953 for (uint32_t i = 0; i < bufferMemoryBarrierCount; i++) {
954 src_flags |= pBufferMemoryBarriers[i].srcAccessMask;
955 dst_flags |= pBufferMemoryBarriers[i].dstAccessMask;
956 }
957
958 for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) {
959 src_flags |= pImageMemoryBarriers[i].srcAccessMask;
960 dst_flags |= pImageMemoryBarriers[i].dstAccessMask;
961 ANV_FROM_HANDLE(anv_image, image, pImageMemoryBarriers[i].image);
962 if (pImageMemoryBarriers[i].subresourceRange.aspectMask &
963 VK_IMAGE_ASPECT_DEPTH_BIT) {
964 transition_depth_buffer(cmd_buffer, image,
965 pImageMemoryBarriers[i].oldLayout,
966 pImageMemoryBarriers[i].newLayout);
967 }
968 }
969
970 enum anv_pipe_bits pipe_bits = 0;
971
972 for_each_bit(b, src_flags) {
973 switch ((VkAccessFlagBits)(1 << b)) {
974 case VK_ACCESS_SHADER_WRITE_BIT:
975 pipe_bits |= ANV_PIPE_DATA_CACHE_FLUSH_BIT;
976 break;
977 case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT:
978 pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
979 break;
980 case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT:
981 pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
982 break;
983 case VK_ACCESS_TRANSFER_WRITE_BIT:
984 pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
985 pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
986 break;
987 default:
988 break; /* Nothing to do */
989 }
990 }
991
992 for_each_bit(b, dst_flags) {
993 switch ((VkAccessFlagBits)(1 << b)) {
994 case VK_ACCESS_INDIRECT_COMMAND_READ_BIT:
995 case VK_ACCESS_INDEX_READ_BIT:
996 case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT:
997 pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
998 break;
999 case VK_ACCESS_UNIFORM_READ_BIT:
1000 pipe_bits |= ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
1001 pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1002 break;
1003 case VK_ACCESS_SHADER_READ_BIT:
1004 case VK_ACCESS_INPUT_ATTACHMENT_READ_BIT:
1005 case VK_ACCESS_TRANSFER_READ_BIT:
1006 pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1007 break;
1008 default:
1009 break; /* Nothing to do */
1010 }
1011 }
1012
1013 cmd_buffer->state.pending_pipe_bits |= pipe_bits;
1014 }
1015
1016 static void
1017 cmd_buffer_alloc_push_constants(struct anv_cmd_buffer *cmd_buffer)
1018 {
1019 VkShaderStageFlags stages = cmd_buffer->state.pipeline->active_stages;
1020
1021 /* In order to avoid thrash, we assume that vertex and fragment stages
1022 * always exist. In the rare case where one is missing *and* the other
1023 * uses push concstants, this may be suboptimal. However, avoiding stalls
1024 * seems more important.
1025 */
1026 stages |= VK_SHADER_STAGE_FRAGMENT_BIT | VK_SHADER_STAGE_VERTEX_BIT;
1027
1028 if (stages == cmd_buffer->state.push_constant_stages)
1029 return;
1030
1031 #if GEN_GEN >= 8
1032 const unsigned push_constant_kb = 32;
1033 #elif GEN_IS_HASWELL
1034 const unsigned push_constant_kb = cmd_buffer->device->info.gt == 3 ? 32 : 16;
1035 #else
1036 const unsigned push_constant_kb = 16;
1037 #endif
1038
1039 const unsigned num_stages =
1040 _mesa_bitcount(stages & VK_SHADER_STAGE_ALL_GRAPHICS);
1041 unsigned size_per_stage = push_constant_kb / num_stages;
1042
1043 /* Broadwell+ and Haswell gt3 require that the push constant sizes be in
1044 * units of 2KB. Incidentally, these are the same platforms that have
1045 * 32KB worth of push constant space.
1046 */
1047 if (push_constant_kb == 32)
1048 size_per_stage &= ~1u;
1049
1050 uint32_t kb_used = 0;
1051 for (int i = MESA_SHADER_VERTEX; i < MESA_SHADER_FRAGMENT; i++) {
1052 unsigned push_size = (stages & (1 << i)) ? size_per_stage : 0;
1053 anv_batch_emit(&cmd_buffer->batch,
1054 GENX(3DSTATE_PUSH_CONSTANT_ALLOC_VS), alloc) {
1055 alloc._3DCommandSubOpcode = 18 + i;
1056 alloc.ConstantBufferOffset = (push_size > 0) ? kb_used : 0;
1057 alloc.ConstantBufferSize = push_size;
1058 }
1059 kb_used += push_size;
1060 }
1061
1062 anv_batch_emit(&cmd_buffer->batch,
1063 GENX(3DSTATE_PUSH_CONSTANT_ALLOC_PS), alloc) {
1064 alloc.ConstantBufferOffset = kb_used;
1065 alloc.ConstantBufferSize = push_constant_kb - kb_used;
1066 }
1067
1068 cmd_buffer->state.push_constant_stages = stages;
1069
1070 /* From the BDW PRM for 3DSTATE_PUSH_CONSTANT_ALLOC_VS:
1071 *
1072 * "The 3DSTATE_CONSTANT_VS must be reprogrammed prior to
1073 * the next 3DPRIMITIVE command after programming the
1074 * 3DSTATE_PUSH_CONSTANT_ALLOC_VS"
1075 *
1076 * Since 3DSTATE_PUSH_CONSTANT_ALLOC_VS is programmed as part of
1077 * pipeline setup, we need to dirty push constants.
1078 */
1079 cmd_buffer->state.push_constants_dirty |= VK_SHADER_STAGE_ALL_GRAPHICS;
1080 }
1081
1082 static VkResult
1083 emit_binding_table(struct anv_cmd_buffer *cmd_buffer,
1084 gl_shader_stage stage,
1085 struct anv_state *bt_state)
1086 {
1087 struct anv_subpass *subpass = cmd_buffer->state.subpass;
1088 struct anv_pipeline *pipeline;
1089 uint32_t bias, state_offset;
1090
1091 switch (stage) {
1092 case MESA_SHADER_COMPUTE:
1093 pipeline = cmd_buffer->state.compute_pipeline;
1094 bias = 1;
1095 break;
1096 default:
1097 pipeline = cmd_buffer->state.pipeline;
1098 bias = 0;
1099 break;
1100 }
1101
1102 if (!anv_pipeline_has_stage(pipeline, stage)) {
1103 *bt_state = (struct anv_state) { 0, };
1104 return VK_SUCCESS;
1105 }
1106
1107 struct anv_pipeline_bind_map *map = &pipeline->shaders[stage]->bind_map;
1108 if (bias + map->surface_count == 0) {
1109 *bt_state = (struct anv_state) { 0, };
1110 return VK_SUCCESS;
1111 }
1112
1113 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer,
1114 bias + map->surface_count,
1115 &state_offset);
1116 uint32_t *bt_map = bt_state->map;
1117
1118 if (bt_state->map == NULL)
1119 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
1120
1121 if (stage == MESA_SHADER_COMPUTE &&
1122 get_cs_prog_data(cmd_buffer->state.compute_pipeline)->uses_num_work_groups) {
1123 struct anv_bo *bo = cmd_buffer->state.num_workgroups_bo;
1124 uint32_t bo_offset = cmd_buffer->state.num_workgroups_offset;
1125
1126 struct anv_state surface_state;
1127 surface_state =
1128 anv_cmd_buffer_alloc_surface_state(cmd_buffer);
1129
1130 const enum isl_format format =
1131 anv_isl_format_for_descriptor_type(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER);
1132 anv_fill_buffer_surface_state(cmd_buffer->device, surface_state,
1133 format, bo_offset, 12, 1);
1134
1135 bt_map[0] = surface_state.offset + state_offset;
1136 add_surface_state_reloc(cmd_buffer, surface_state, bo, bo_offset);
1137 }
1138
1139 if (map->surface_count == 0)
1140 goto out;
1141
1142 if (map->image_count > 0) {
1143 VkResult result =
1144 anv_cmd_buffer_ensure_push_constant_field(cmd_buffer, stage, images);
1145 if (result != VK_SUCCESS)
1146 return result;
1147
1148 cmd_buffer->state.push_constants_dirty |= 1 << stage;
1149 }
1150
1151 uint32_t image = 0;
1152 for (uint32_t s = 0; s < map->surface_count; s++) {
1153 struct anv_pipeline_binding *binding = &map->surface_to_descriptor[s];
1154
1155 struct anv_state surface_state;
1156
1157 if (binding->set == ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS) {
1158 /* Color attachment binding */
1159 assert(stage == MESA_SHADER_FRAGMENT);
1160 assert(binding->binding == 0);
1161 if (binding->index < subpass->color_count) {
1162 const unsigned att = subpass->color_attachments[binding->index].attachment;
1163 surface_state = cmd_buffer->state.attachments[att].color_rt_state;
1164 } else {
1165 surface_state = cmd_buffer->state.null_surface_state;
1166 }
1167
1168 bt_map[bias + s] = surface_state.offset + state_offset;
1169 continue;
1170 }
1171
1172 struct anv_descriptor_set *set =
1173 cmd_buffer->state.descriptors[binding->set];
1174 uint32_t offset = set->layout->binding[binding->binding].descriptor_index;
1175 struct anv_descriptor *desc = &set->descriptors[offset + binding->index];
1176
1177 switch (desc->type) {
1178 case VK_DESCRIPTOR_TYPE_SAMPLER:
1179 /* Nothing for us to do here */
1180 continue;
1181
1182 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1183 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1184 surface_state = desc->aux_usage == ISL_AUX_USAGE_NONE ?
1185 desc->image_view->no_aux_sampler_surface_state :
1186 desc->image_view->sampler_surface_state;
1187 assert(surface_state.alloc_size);
1188 add_image_view_relocs(cmd_buffer, desc->image_view,
1189 desc->aux_usage, surface_state);
1190 break;
1191 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1192 assert(stage == MESA_SHADER_FRAGMENT);
1193 if (desc->image_view->aspect_mask != VK_IMAGE_ASPECT_COLOR_BIT) {
1194 /* For depth and stencil input attachments, we treat it like any
1195 * old texture that a user may have bound.
1196 */
1197 surface_state = desc->aux_usage == ISL_AUX_USAGE_NONE ?
1198 desc->image_view->no_aux_sampler_surface_state :
1199 desc->image_view->sampler_surface_state;
1200 assert(surface_state.alloc_size);
1201 add_image_view_relocs(cmd_buffer, desc->image_view,
1202 desc->aux_usage, surface_state);
1203 } else {
1204 /* For color input attachments, we create the surface state at
1205 * vkBeginRenderPass time so that we can include aux and clear
1206 * color information.
1207 */
1208 assert(binding->input_attachment_index < subpass->input_count);
1209 const unsigned subpass_att = binding->input_attachment_index;
1210 const unsigned att = subpass->input_attachments[subpass_att].attachment;
1211 surface_state = cmd_buffer->state.attachments[att].input_att_state;
1212 }
1213 break;
1214
1215 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: {
1216 surface_state = (binding->write_only)
1217 ? desc->image_view->writeonly_storage_surface_state
1218 : desc->image_view->storage_surface_state;
1219 assert(surface_state.alloc_size);
1220 add_image_view_relocs(cmd_buffer, desc->image_view,
1221 desc->image_view->image->aux_usage,
1222 surface_state);
1223
1224 struct brw_image_param *image_param =
1225 &cmd_buffer->state.push_constants[stage]->images[image++];
1226
1227 *image_param = desc->image_view->storage_image_param;
1228 image_param->surface_idx = bias + s;
1229 break;
1230 }
1231
1232 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1233 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1234 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
1235 surface_state = desc->buffer_view->surface_state;
1236 assert(surface_state.alloc_size);
1237 add_surface_state_reloc(cmd_buffer, surface_state,
1238 desc->buffer_view->bo,
1239 desc->buffer_view->offset);
1240 break;
1241
1242 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1243 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1244 uint32_t dynamic_offset_idx =
1245 pipeline->layout->set[binding->set].dynamic_offset_start +
1246 set->layout->binding[binding->binding].dynamic_offset_index +
1247 binding->index;
1248
1249 /* Compute the offset within the buffer */
1250 uint64_t offset = desc->offset +
1251 cmd_buffer->state.dynamic_offsets[dynamic_offset_idx];
1252 /* Clamp to the buffer size */
1253 offset = MIN2(offset, desc->buffer->size);
1254 /* Clamp the range to the buffer size */
1255 uint32_t range = MIN2(desc->range, desc->buffer->size - offset);
1256
1257 surface_state =
1258 anv_state_stream_alloc(&cmd_buffer->surface_state_stream, 64, 64);
1259 enum isl_format format =
1260 anv_isl_format_for_descriptor_type(desc->type);
1261
1262 anv_fill_buffer_surface_state(cmd_buffer->device, surface_state,
1263 format, offset, range, 1);
1264 add_surface_state_reloc(cmd_buffer, surface_state,
1265 desc->buffer->bo,
1266 desc->buffer->offset + offset);
1267 break;
1268 }
1269
1270 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
1271 surface_state = (binding->write_only)
1272 ? desc->buffer_view->writeonly_storage_surface_state
1273 : desc->buffer_view->storage_surface_state;
1274 assert(surface_state.alloc_size);
1275 add_surface_state_reloc(cmd_buffer, surface_state,
1276 desc->buffer_view->bo,
1277 desc->buffer_view->offset);
1278
1279 struct brw_image_param *image_param =
1280 &cmd_buffer->state.push_constants[stage]->images[image++];
1281
1282 *image_param = desc->buffer_view->storage_image_param;
1283 image_param->surface_idx = bias + s;
1284 break;
1285
1286 default:
1287 assert(!"Invalid descriptor type");
1288 continue;
1289 }
1290
1291 bt_map[bias + s] = surface_state.offset + state_offset;
1292 }
1293 assert(image == map->image_count);
1294
1295 out:
1296 anv_state_flush(cmd_buffer->device, *bt_state);
1297
1298 return VK_SUCCESS;
1299 }
1300
1301 static VkResult
1302 emit_samplers(struct anv_cmd_buffer *cmd_buffer,
1303 gl_shader_stage stage,
1304 struct anv_state *state)
1305 {
1306 struct anv_pipeline *pipeline;
1307
1308 if (stage == MESA_SHADER_COMPUTE)
1309 pipeline = cmd_buffer->state.compute_pipeline;
1310 else
1311 pipeline = cmd_buffer->state.pipeline;
1312
1313 if (!anv_pipeline_has_stage(pipeline, stage)) {
1314 *state = (struct anv_state) { 0, };
1315 return VK_SUCCESS;
1316 }
1317
1318 struct anv_pipeline_bind_map *map = &pipeline->shaders[stage]->bind_map;
1319 if (map->sampler_count == 0) {
1320 *state = (struct anv_state) { 0, };
1321 return VK_SUCCESS;
1322 }
1323
1324 uint32_t size = map->sampler_count * 16;
1325 *state = anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, size, 32);
1326
1327 if (state->map == NULL)
1328 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
1329
1330 for (uint32_t s = 0; s < map->sampler_count; s++) {
1331 struct anv_pipeline_binding *binding = &map->sampler_to_descriptor[s];
1332 struct anv_descriptor_set *set =
1333 cmd_buffer->state.descriptors[binding->set];
1334 uint32_t offset = set->layout->binding[binding->binding].descriptor_index;
1335 struct anv_descriptor *desc = &set->descriptors[offset + binding->index];
1336
1337 if (desc->type != VK_DESCRIPTOR_TYPE_SAMPLER &&
1338 desc->type != VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
1339 continue;
1340
1341 struct anv_sampler *sampler = desc->sampler;
1342
1343 /* This can happen if we have an unfilled slot since TYPE_SAMPLER
1344 * happens to be zero.
1345 */
1346 if (sampler == NULL)
1347 continue;
1348
1349 memcpy(state->map + (s * 16),
1350 sampler->state, sizeof(sampler->state));
1351 }
1352
1353 anv_state_flush(cmd_buffer->device, *state);
1354
1355 return VK_SUCCESS;
1356 }
1357
1358 static uint32_t
1359 flush_descriptor_sets(struct anv_cmd_buffer *cmd_buffer)
1360 {
1361 VkShaderStageFlags dirty = cmd_buffer->state.descriptors_dirty &
1362 cmd_buffer->state.pipeline->active_stages;
1363
1364 VkResult result = VK_SUCCESS;
1365 anv_foreach_stage(s, dirty) {
1366 result = emit_samplers(cmd_buffer, s, &cmd_buffer->state.samplers[s]);
1367 if (result != VK_SUCCESS)
1368 break;
1369 result = emit_binding_table(cmd_buffer, s,
1370 &cmd_buffer->state.binding_tables[s]);
1371 if (result != VK_SUCCESS)
1372 break;
1373 }
1374
1375 if (result != VK_SUCCESS) {
1376 assert(result == VK_ERROR_OUT_OF_DEVICE_MEMORY);
1377
1378 result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
1379 assert(result == VK_SUCCESS);
1380
1381 /* Re-emit state base addresses so we get the new surface state base
1382 * address before we start emitting binding tables etc.
1383 */
1384 genX(cmd_buffer_emit_state_base_address)(cmd_buffer);
1385
1386 /* Re-emit all active binding tables */
1387 dirty |= cmd_buffer->state.pipeline->active_stages;
1388 anv_foreach_stage(s, dirty) {
1389 result = emit_samplers(cmd_buffer, s, &cmd_buffer->state.samplers[s]);
1390 if (result != VK_SUCCESS)
1391 return result;
1392 result = emit_binding_table(cmd_buffer, s,
1393 &cmd_buffer->state.binding_tables[s]);
1394 if (result != VK_SUCCESS)
1395 return result;
1396 }
1397 }
1398
1399 cmd_buffer->state.descriptors_dirty &= ~dirty;
1400
1401 return dirty;
1402 }
1403
1404 static void
1405 cmd_buffer_emit_descriptor_pointers(struct anv_cmd_buffer *cmd_buffer,
1406 uint32_t stages)
1407 {
1408 static const uint32_t sampler_state_opcodes[] = {
1409 [MESA_SHADER_VERTEX] = 43,
1410 [MESA_SHADER_TESS_CTRL] = 44, /* HS */
1411 [MESA_SHADER_TESS_EVAL] = 45, /* DS */
1412 [MESA_SHADER_GEOMETRY] = 46,
1413 [MESA_SHADER_FRAGMENT] = 47,
1414 [MESA_SHADER_COMPUTE] = 0,
1415 };
1416
1417 static const uint32_t binding_table_opcodes[] = {
1418 [MESA_SHADER_VERTEX] = 38,
1419 [MESA_SHADER_TESS_CTRL] = 39,
1420 [MESA_SHADER_TESS_EVAL] = 40,
1421 [MESA_SHADER_GEOMETRY] = 41,
1422 [MESA_SHADER_FRAGMENT] = 42,
1423 [MESA_SHADER_COMPUTE] = 0,
1424 };
1425
1426 anv_foreach_stage(s, stages) {
1427 if (cmd_buffer->state.samplers[s].alloc_size > 0) {
1428 anv_batch_emit(&cmd_buffer->batch,
1429 GENX(3DSTATE_SAMPLER_STATE_POINTERS_VS), ssp) {
1430 ssp._3DCommandSubOpcode = sampler_state_opcodes[s];
1431 ssp.PointertoVSSamplerState = cmd_buffer->state.samplers[s].offset;
1432 }
1433 }
1434
1435 /* Always emit binding table pointers if we're asked to, since on SKL
1436 * this is what flushes push constants. */
1437 anv_batch_emit(&cmd_buffer->batch,
1438 GENX(3DSTATE_BINDING_TABLE_POINTERS_VS), btp) {
1439 btp._3DCommandSubOpcode = binding_table_opcodes[s];
1440 btp.PointertoVSBindingTable = cmd_buffer->state.binding_tables[s].offset;
1441 }
1442 }
1443 }
1444
1445 static uint32_t
1446 cmd_buffer_flush_push_constants(struct anv_cmd_buffer *cmd_buffer)
1447 {
1448 static const uint32_t push_constant_opcodes[] = {
1449 [MESA_SHADER_VERTEX] = 21,
1450 [MESA_SHADER_TESS_CTRL] = 25, /* HS */
1451 [MESA_SHADER_TESS_EVAL] = 26, /* DS */
1452 [MESA_SHADER_GEOMETRY] = 22,
1453 [MESA_SHADER_FRAGMENT] = 23,
1454 [MESA_SHADER_COMPUTE] = 0,
1455 };
1456
1457 VkShaderStageFlags flushed = 0;
1458
1459 anv_foreach_stage(stage, cmd_buffer->state.push_constants_dirty) {
1460 if (stage == MESA_SHADER_COMPUTE)
1461 continue;
1462
1463 struct anv_state state = anv_cmd_buffer_push_constants(cmd_buffer, stage);
1464
1465 if (state.offset == 0) {
1466 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CONSTANT_VS), c)
1467 c._3DCommandSubOpcode = push_constant_opcodes[stage];
1468 } else {
1469 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CONSTANT_VS), c) {
1470 c._3DCommandSubOpcode = push_constant_opcodes[stage],
1471 c.ConstantBody = (struct GENX(3DSTATE_CONSTANT_BODY)) {
1472 #if GEN_GEN >= 9
1473 .PointerToConstantBuffer2 = { &cmd_buffer->device->dynamic_state_block_pool.bo, state.offset },
1474 .ConstantBuffer2ReadLength = DIV_ROUND_UP(state.alloc_size, 32),
1475 #else
1476 .PointerToConstantBuffer0 = { .offset = state.offset },
1477 .ConstantBuffer0ReadLength = DIV_ROUND_UP(state.alloc_size, 32),
1478 #endif
1479 };
1480 }
1481 }
1482
1483 flushed |= mesa_to_vk_shader_stage(stage);
1484 }
1485
1486 cmd_buffer->state.push_constants_dirty &= ~VK_SHADER_STAGE_ALL_GRAPHICS;
1487
1488 return flushed;
1489 }
1490
1491 void
1492 genX(cmd_buffer_flush_state)(struct anv_cmd_buffer *cmd_buffer)
1493 {
1494 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
1495 uint32_t *p;
1496
1497 uint32_t vb_emit = cmd_buffer->state.vb_dirty & pipeline->vb_used;
1498
1499 assert((pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT) == 0);
1500
1501 genX(cmd_buffer_config_l3)(cmd_buffer, pipeline->urb.l3_config);
1502
1503 genX(flush_pipeline_select_3d)(cmd_buffer);
1504
1505 if (vb_emit) {
1506 const uint32_t num_buffers = __builtin_popcount(vb_emit);
1507 const uint32_t num_dwords = 1 + num_buffers * 4;
1508
1509 p = anv_batch_emitn(&cmd_buffer->batch, num_dwords,
1510 GENX(3DSTATE_VERTEX_BUFFERS));
1511 uint32_t vb, i = 0;
1512 for_each_bit(vb, vb_emit) {
1513 struct anv_buffer *buffer = cmd_buffer->state.vertex_bindings[vb].buffer;
1514 uint32_t offset = cmd_buffer->state.vertex_bindings[vb].offset;
1515
1516 struct GENX(VERTEX_BUFFER_STATE) state = {
1517 .VertexBufferIndex = vb,
1518
1519 #if GEN_GEN >= 8
1520 .MemoryObjectControlState = GENX(MOCS),
1521 #else
1522 .BufferAccessType = pipeline->instancing_enable[vb] ? INSTANCEDATA : VERTEXDATA,
1523 .InstanceDataStepRate = 1,
1524 .VertexBufferMemoryObjectControlState = GENX(MOCS),
1525 #endif
1526
1527 .AddressModifyEnable = true,
1528 .BufferPitch = pipeline->binding_stride[vb],
1529 .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
1530
1531 #if GEN_GEN >= 8
1532 .BufferSize = buffer->size - offset
1533 #else
1534 .EndAddress = { buffer->bo, buffer->offset + buffer->size - 1},
1535 #endif
1536 };
1537
1538 GENX(VERTEX_BUFFER_STATE_pack)(&cmd_buffer->batch, &p[1 + i * 4], &state);
1539 i++;
1540 }
1541 }
1542
1543 cmd_buffer->state.vb_dirty &= ~vb_emit;
1544
1545 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_PIPELINE) {
1546 anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
1547
1548 /* The exact descriptor layout is pulled from the pipeline, so we need
1549 * to re-emit binding tables on every pipeline change.
1550 */
1551 cmd_buffer->state.descriptors_dirty |=
1552 cmd_buffer->state.pipeline->active_stages;
1553
1554 /* If the pipeline changed, we may need to re-allocate push constant
1555 * space in the URB.
1556 */
1557 cmd_buffer_alloc_push_constants(cmd_buffer);
1558 }
1559
1560 #if GEN_GEN <= 7
1561 if (cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_VERTEX_BIT ||
1562 cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_VERTEX_BIT) {
1563 /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
1564 *
1565 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
1566 * stall needs to be sent just prior to any 3DSTATE_VS,
1567 * 3DSTATE_URB_VS, 3DSTATE_CONSTANT_VS,
1568 * 3DSTATE_BINDING_TABLE_POINTER_VS,
1569 * 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one
1570 * PIPE_CONTROL needs to be sent before any combination of VS
1571 * associated 3DSTATE."
1572 */
1573 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
1574 pc.DepthStallEnable = true;
1575 pc.PostSyncOperation = WriteImmediateData;
1576 pc.Address =
1577 (struct anv_address) { &cmd_buffer->device->workaround_bo, 0 };
1578 }
1579 }
1580 #endif
1581
1582 /* Render targets live in the same binding table as fragment descriptors */
1583 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_RENDER_TARGETS)
1584 cmd_buffer->state.descriptors_dirty |= VK_SHADER_STAGE_FRAGMENT_BIT;
1585
1586 /* We emit the binding tables and sampler tables first, then emit push
1587 * constants and then finally emit binding table and sampler table
1588 * pointers. It has to happen in this order, since emitting the binding
1589 * tables may change the push constants (in case of storage images). After
1590 * emitting push constants, on SKL+ we have to emit the corresponding
1591 * 3DSTATE_BINDING_TABLE_POINTER_* for the push constants to take effect.
1592 */
1593 uint32_t dirty = 0;
1594 if (cmd_buffer->state.descriptors_dirty)
1595 dirty = flush_descriptor_sets(cmd_buffer);
1596
1597 if (cmd_buffer->state.push_constants_dirty) {
1598 #if GEN_GEN >= 9
1599 /* On Sky Lake and later, the binding table pointers commands are
1600 * what actually flush the changes to push constant state so we need
1601 * to dirty them so they get re-emitted below.
1602 */
1603 dirty |= cmd_buffer_flush_push_constants(cmd_buffer);
1604 #else
1605 cmd_buffer_flush_push_constants(cmd_buffer);
1606 #endif
1607 }
1608
1609 if (dirty)
1610 cmd_buffer_emit_descriptor_pointers(cmd_buffer, dirty);
1611
1612 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_VIEWPORT)
1613 gen8_cmd_buffer_emit_viewport(cmd_buffer);
1614
1615 if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_DYNAMIC_VIEWPORT |
1616 ANV_CMD_DIRTY_PIPELINE)) {
1617 gen8_cmd_buffer_emit_depth_viewport(cmd_buffer,
1618 pipeline->depth_clamp_enable);
1619 }
1620
1621 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_SCISSOR)
1622 gen7_cmd_buffer_emit_scissor(cmd_buffer);
1623
1624 genX(cmd_buffer_flush_dynamic_state)(cmd_buffer);
1625
1626 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
1627 }
1628
1629 static void
1630 emit_vertex_bo(struct anv_cmd_buffer *cmd_buffer,
1631 struct anv_bo *bo, uint32_t offset,
1632 uint32_t size, uint32_t index)
1633 {
1634 uint32_t *p = anv_batch_emitn(&cmd_buffer->batch, 5,
1635 GENX(3DSTATE_VERTEX_BUFFERS));
1636
1637 GENX(VERTEX_BUFFER_STATE_pack)(&cmd_buffer->batch, p + 1,
1638 &(struct GENX(VERTEX_BUFFER_STATE)) {
1639 .VertexBufferIndex = index,
1640 .AddressModifyEnable = true,
1641 .BufferPitch = 0,
1642 #if (GEN_GEN >= 8)
1643 .MemoryObjectControlState = GENX(MOCS),
1644 .BufferStartingAddress = { bo, offset },
1645 .BufferSize = size
1646 #else
1647 .VertexBufferMemoryObjectControlState = GENX(MOCS),
1648 .BufferStartingAddress = { bo, offset },
1649 .EndAddress = { bo, offset + size },
1650 #endif
1651 });
1652 }
1653
1654 static void
1655 emit_base_vertex_instance_bo(struct anv_cmd_buffer *cmd_buffer,
1656 struct anv_bo *bo, uint32_t offset)
1657 {
1658 emit_vertex_bo(cmd_buffer, bo, offset, 8, ANV_SVGS_VB_INDEX);
1659 }
1660
1661 static void
1662 emit_base_vertex_instance(struct anv_cmd_buffer *cmd_buffer,
1663 uint32_t base_vertex, uint32_t base_instance)
1664 {
1665 struct anv_state id_state =
1666 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, 8, 4);
1667
1668 ((uint32_t *)id_state.map)[0] = base_vertex;
1669 ((uint32_t *)id_state.map)[1] = base_instance;
1670
1671 anv_state_flush(cmd_buffer->device, id_state);
1672
1673 emit_base_vertex_instance_bo(cmd_buffer,
1674 &cmd_buffer->device->dynamic_state_block_pool.bo, id_state.offset);
1675 }
1676
1677 static void
1678 emit_draw_index(struct anv_cmd_buffer *cmd_buffer, uint32_t draw_index)
1679 {
1680 struct anv_state state =
1681 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, 4, 4);
1682
1683 ((uint32_t *)state.map)[0] = draw_index;
1684
1685 anv_state_flush(cmd_buffer->device, state);
1686
1687 emit_vertex_bo(cmd_buffer,
1688 &cmd_buffer->device->dynamic_state_block_pool.bo,
1689 state.offset, 4, ANV_DRAWID_VB_INDEX);
1690 }
1691
1692 void genX(CmdDraw)(
1693 VkCommandBuffer commandBuffer,
1694 uint32_t vertexCount,
1695 uint32_t instanceCount,
1696 uint32_t firstVertex,
1697 uint32_t firstInstance)
1698 {
1699 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1700 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
1701 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
1702
1703 if (anv_batch_has_error(&cmd_buffer->batch))
1704 return;
1705
1706 genX(cmd_buffer_flush_state)(cmd_buffer);
1707
1708 if (vs_prog_data->uses_basevertex || vs_prog_data->uses_baseinstance)
1709 emit_base_vertex_instance(cmd_buffer, firstVertex, firstInstance);
1710 if (vs_prog_data->uses_drawid)
1711 emit_draw_index(cmd_buffer, 0);
1712
1713 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
1714 prim.VertexAccessType = SEQUENTIAL;
1715 prim.PrimitiveTopologyType = pipeline->topology;
1716 prim.VertexCountPerInstance = vertexCount;
1717 prim.StartVertexLocation = firstVertex;
1718 prim.InstanceCount = instanceCount;
1719 prim.StartInstanceLocation = firstInstance;
1720 prim.BaseVertexLocation = 0;
1721 }
1722 }
1723
1724 void genX(CmdDrawIndexed)(
1725 VkCommandBuffer commandBuffer,
1726 uint32_t indexCount,
1727 uint32_t instanceCount,
1728 uint32_t firstIndex,
1729 int32_t vertexOffset,
1730 uint32_t firstInstance)
1731 {
1732 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1733 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
1734 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
1735
1736 if (anv_batch_has_error(&cmd_buffer->batch))
1737 return;
1738
1739 genX(cmd_buffer_flush_state)(cmd_buffer);
1740
1741 if (vs_prog_data->uses_basevertex || vs_prog_data->uses_baseinstance)
1742 emit_base_vertex_instance(cmd_buffer, vertexOffset, firstInstance);
1743 if (vs_prog_data->uses_drawid)
1744 emit_draw_index(cmd_buffer, 0);
1745
1746 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
1747 prim.VertexAccessType = RANDOM;
1748 prim.PrimitiveTopologyType = pipeline->topology;
1749 prim.VertexCountPerInstance = indexCount;
1750 prim.StartVertexLocation = firstIndex;
1751 prim.InstanceCount = instanceCount;
1752 prim.StartInstanceLocation = firstInstance;
1753 prim.BaseVertexLocation = vertexOffset;
1754 }
1755 }
1756
1757 /* Auto-Draw / Indirect Registers */
1758 #define GEN7_3DPRIM_END_OFFSET 0x2420
1759 #define GEN7_3DPRIM_START_VERTEX 0x2430
1760 #define GEN7_3DPRIM_VERTEX_COUNT 0x2434
1761 #define GEN7_3DPRIM_INSTANCE_COUNT 0x2438
1762 #define GEN7_3DPRIM_START_INSTANCE 0x243C
1763 #define GEN7_3DPRIM_BASE_VERTEX 0x2440
1764
1765 void genX(CmdDrawIndirect)(
1766 VkCommandBuffer commandBuffer,
1767 VkBuffer _buffer,
1768 VkDeviceSize offset,
1769 uint32_t drawCount,
1770 uint32_t stride)
1771 {
1772 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1773 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
1774 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
1775 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
1776 struct anv_bo *bo = buffer->bo;
1777 uint32_t bo_offset = buffer->offset + offset;
1778
1779 if (anv_batch_has_error(&cmd_buffer->batch))
1780 return;
1781
1782 genX(cmd_buffer_flush_state)(cmd_buffer);
1783
1784 if (vs_prog_data->uses_basevertex || vs_prog_data->uses_baseinstance)
1785 emit_base_vertex_instance_bo(cmd_buffer, bo, bo_offset + 8);
1786 if (vs_prog_data->uses_drawid)
1787 emit_draw_index(cmd_buffer, 0);
1788
1789 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
1790 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
1791 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
1792 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 12);
1793 emit_lri(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, 0);
1794
1795 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
1796 prim.IndirectParameterEnable = true;
1797 prim.VertexAccessType = SEQUENTIAL;
1798 prim.PrimitiveTopologyType = pipeline->topology;
1799 }
1800 }
1801
1802 void genX(CmdDrawIndexedIndirect)(
1803 VkCommandBuffer commandBuffer,
1804 VkBuffer _buffer,
1805 VkDeviceSize offset,
1806 uint32_t drawCount,
1807 uint32_t stride)
1808 {
1809 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1810 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
1811 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
1812 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
1813 struct anv_bo *bo = buffer->bo;
1814 uint32_t bo_offset = buffer->offset + offset;
1815
1816 if (anv_batch_has_error(&cmd_buffer->batch))
1817 return;
1818
1819 genX(cmd_buffer_flush_state)(cmd_buffer);
1820
1821 /* TODO: We need to stomp base vertex to 0 somehow */
1822 if (vs_prog_data->uses_basevertex || vs_prog_data->uses_baseinstance)
1823 emit_base_vertex_instance_bo(cmd_buffer, bo, bo_offset + 12);
1824 if (vs_prog_data->uses_drawid)
1825 emit_draw_index(cmd_buffer, 0);
1826
1827 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
1828 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
1829 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
1830 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, bo, bo_offset + 12);
1831 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 16);
1832
1833 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
1834 prim.IndirectParameterEnable = true;
1835 prim.VertexAccessType = RANDOM;
1836 prim.PrimitiveTopologyType = pipeline->topology;
1837 }
1838 }
1839
1840 static VkResult
1841 flush_compute_descriptor_set(struct anv_cmd_buffer *cmd_buffer)
1842 {
1843 struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
1844 struct anv_state surfaces = { 0, }, samplers = { 0, };
1845 VkResult result;
1846
1847 result = emit_binding_table(cmd_buffer, MESA_SHADER_COMPUTE, &surfaces);
1848 if (result != VK_SUCCESS) {
1849 assert(result == VK_ERROR_OUT_OF_DEVICE_MEMORY);
1850 result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
1851 assert(result == VK_SUCCESS);
1852
1853 /* Re-emit state base addresses so we get the new surface state base
1854 * address before we start emitting binding tables etc.
1855 */
1856 genX(cmd_buffer_emit_state_base_address)(cmd_buffer);
1857
1858 result = emit_binding_table(cmd_buffer, MESA_SHADER_COMPUTE, &surfaces);
1859 assert(result == VK_SUCCESS);
1860 }
1861
1862 result = emit_samplers(cmd_buffer, MESA_SHADER_COMPUTE, &samplers);
1863 assert(result == VK_SUCCESS);
1864
1865 uint32_t iface_desc_data_dw[GENX(INTERFACE_DESCRIPTOR_DATA_length)];
1866 struct GENX(INTERFACE_DESCRIPTOR_DATA) desc = {
1867 .BindingTablePointer = surfaces.offset,
1868 .SamplerStatePointer = samplers.offset,
1869 };
1870 GENX(INTERFACE_DESCRIPTOR_DATA_pack)(NULL, iface_desc_data_dw, &desc);
1871
1872 struct anv_state state =
1873 anv_cmd_buffer_merge_dynamic(cmd_buffer, iface_desc_data_dw,
1874 pipeline->interface_descriptor_data,
1875 GENX(INTERFACE_DESCRIPTOR_DATA_length),
1876 64);
1877
1878 uint32_t size = GENX(INTERFACE_DESCRIPTOR_DATA_length) * sizeof(uint32_t);
1879 anv_batch_emit(&cmd_buffer->batch,
1880 GENX(MEDIA_INTERFACE_DESCRIPTOR_LOAD), mid) {
1881 mid.InterfaceDescriptorTotalLength = size;
1882 mid.InterfaceDescriptorDataStartAddress = state.offset;
1883 }
1884
1885 return VK_SUCCESS;
1886 }
1887
1888 void
1889 genX(cmd_buffer_flush_compute_state)(struct anv_cmd_buffer *cmd_buffer)
1890 {
1891 struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
1892 MAYBE_UNUSED VkResult result;
1893
1894 assert(pipeline->active_stages == VK_SHADER_STAGE_COMPUTE_BIT);
1895
1896 genX(cmd_buffer_config_l3)(cmd_buffer, pipeline->urb.l3_config);
1897
1898 genX(flush_pipeline_select_gpgpu)(cmd_buffer);
1899
1900 if (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE) {
1901 /* From the Sky Lake PRM Vol 2a, MEDIA_VFE_STATE:
1902 *
1903 * "A stalling PIPE_CONTROL is required before MEDIA_VFE_STATE unless
1904 * the only bits that are changed are scoreboard related: Scoreboard
1905 * Enable, Scoreboard Type, Scoreboard Mask, Scoreboard * Delta. For
1906 * these scoreboard related states, a MEDIA_STATE_FLUSH is
1907 * sufficient."
1908 */
1909 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_CS_STALL_BIT;
1910 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
1911
1912 anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
1913 }
1914
1915 if ((cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_COMPUTE_BIT) ||
1916 (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)) {
1917 /* FIXME: figure out descriptors for gen7 */
1918 result = flush_compute_descriptor_set(cmd_buffer);
1919 assert(result == VK_SUCCESS);
1920 cmd_buffer->state.descriptors_dirty &= ~VK_SHADER_STAGE_COMPUTE_BIT;
1921 }
1922
1923 if (cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_COMPUTE_BIT) {
1924 struct anv_state push_state =
1925 anv_cmd_buffer_cs_push_constants(cmd_buffer);
1926
1927 if (push_state.alloc_size) {
1928 anv_batch_emit(&cmd_buffer->batch, GENX(MEDIA_CURBE_LOAD), curbe) {
1929 curbe.CURBETotalDataLength = push_state.alloc_size;
1930 curbe.CURBEDataStartAddress = push_state.offset;
1931 }
1932 }
1933 }
1934
1935 cmd_buffer->state.compute_dirty = 0;
1936
1937 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
1938 }
1939
1940 #if GEN_GEN == 7
1941
1942 static VkResult
1943 verify_cmd_parser(const struct anv_device *device,
1944 int required_version,
1945 const char *function)
1946 {
1947 if (device->instance->physicalDevice.cmd_parser_version < required_version) {
1948 return vk_errorf(VK_ERROR_FEATURE_NOT_PRESENT,
1949 "cmd parser version %d is required for %s",
1950 required_version, function);
1951 } else {
1952 return VK_SUCCESS;
1953 }
1954 }
1955
1956 #endif
1957
1958 void genX(CmdDispatch)(
1959 VkCommandBuffer commandBuffer,
1960 uint32_t x,
1961 uint32_t y,
1962 uint32_t z)
1963 {
1964 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1965 struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
1966 const struct brw_cs_prog_data *prog_data = get_cs_prog_data(pipeline);
1967
1968 if (anv_batch_has_error(&cmd_buffer->batch))
1969 return;
1970
1971 if (prog_data->uses_num_work_groups) {
1972 struct anv_state state =
1973 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, 12, 4);
1974 uint32_t *sizes = state.map;
1975 sizes[0] = x;
1976 sizes[1] = y;
1977 sizes[2] = z;
1978 anv_state_flush(cmd_buffer->device, state);
1979 cmd_buffer->state.num_workgroups_offset = state.offset;
1980 cmd_buffer->state.num_workgroups_bo =
1981 &cmd_buffer->device->dynamic_state_block_pool.bo;
1982 }
1983
1984 genX(cmd_buffer_flush_compute_state)(cmd_buffer);
1985
1986 anv_batch_emit(&cmd_buffer->batch, GENX(GPGPU_WALKER), ggw) {
1987 ggw.SIMDSize = prog_data->simd_size / 16;
1988 ggw.ThreadDepthCounterMaximum = 0;
1989 ggw.ThreadHeightCounterMaximum = 0;
1990 ggw.ThreadWidthCounterMaximum = prog_data->threads - 1;
1991 ggw.ThreadGroupIDXDimension = x;
1992 ggw.ThreadGroupIDYDimension = y;
1993 ggw.ThreadGroupIDZDimension = z;
1994 ggw.RightExecutionMask = pipeline->cs_right_mask;
1995 ggw.BottomExecutionMask = 0xffffffff;
1996 }
1997
1998 anv_batch_emit(&cmd_buffer->batch, GENX(MEDIA_STATE_FLUSH), msf);
1999 }
2000
2001 #define GPGPU_DISPATCHDIMX 0x2500
2002 #define GPGPU_DISPATCHDIMY 0x2504
2003 #define GPGPU_DISPATCHDIMZ 0x2508
2004
2005 #define MI_PREDICATE_SRC0 0x2400
2006 #define MI_PREDICATE_SRC1 0x2408
2007
2008 void genX(CmdDispatchIndirect)(
2009 VkCommandBuffer commandBuffer,
2010 VkBuffer _buffer,
2011 VkDeviceSize offset)
2012 {
2013 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
2014 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
2015 struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
2016 const struct brw_cs_prog_data *prog_data = get_cs_prog_data(pipeline);
2017 struct anv_bo *bo = buffer->bo;
2018 uint32_t bo_offset = buffer->offset + offset;
2019 struct anv_batch *batch = &cmd_buffer->batch;
2020
2021 #if GEN_GEN == 7
2022 /* Linux 4.4 added command parser version 5 which allows the GPGPU
2023 * indirect dispatch registers to be written.
2024 */
2025 if (verify_cmd_parser(cmd_buffer->device, 5,
2026 "vkCmdDispatchIndirect") != VK_SUCCESS)
2027 return;
2028 #endif
2029
2030 if (prog_data->uses_num_work_groups) {
2031 cmd_buffer->state.num_workgroups_offset = bo_offset;
2032 cmd_buffer->state.num_workgroups_bo = bo;
2033 }
2034
2035 genX(cmd_buffer_flush_compute_state)(cmd_buffer);
2036
2037 emit_lrm(batch, GPGPU_DISPATCHDIMX, bo, bo_offset);
2038 emit_lrm(batch, GPGPU_DISPATCHDIMY, bo, bo_offset + 4);
2039 emit_lrm(batch, GPGPU_DISPATCHDIMZ, bo, bo_offset + 8);
2040
2041 #if GEN_GEN <= 7
2042 /* Clear upper 32-bits of SRC0 and all 64-bits of SRC1 */
2043 emit_lri(batch, MI_PREDICATE_SRC0 + 4, 0);
2044 emit_lri(batch, MI_PREDICATE_SRC1 + 0, 0);
2045 emit_lri(batch, MI_PREDICATE_SRC1 + 4, 0);
2046
2047 /* Load compute_dispatch_indirect_x_size into SRC0 */
2048 emit_lrm(batch, MI_PREDICATE_SRC0, bo, bo_offset + 0);
2049
2050 /* predicate = (compute_dispatch_indirect_x_size == 0); */
2051 anv_batch_emit(batch, GENX(MI_PREDICATE), mip) {
2052 mip.LoadOperation = LOAD_LOAD;
2053 mip.CombineOperation = COMBINE_SET;
2054 mip.CompareOperation = COMPARE_SRCS_EQUAL;
2055 }
2056
2057 /* Load compute_dispatch_indirect_y_size into SRC0 */
2058 emit_lrm(batch, MI_PREDICATE_SRC0, bo, bo_offset + 4);
2059
2060 /* predicate |= (compute_dispatch_indirect_y_size == 0); */
2061 anv_batch_emit(batch, GENX(MI_PREDICATE), mip) {
2062 mip.LoadOperation = LOAD_LOAD;
2063 mip.CombineOperation = COMBINE_OR;
2064 mip.CompareOperation = COMPARE_SRCS_EQUAL;
2065 }
2066
2067 /* Load compute_dispatch_indirect_z_size into SRC0 */
2068 emit_lrm(batch, MI_PREDICATE_SRC0, bo, bo_offset + 8);
2069
2070 /* predicate |= (compute_dispatch_indirect_z_size == 0); */
2071 anv_batch_emit(batch, GENX(MI_PREDICATE), mip) {
2072 mip.LoadOperation = LOAD_LOAD;
2073 mip.CombineOperation = COMBINE_OR;
2074 mip.CompareOperation = COMPARE_SRCS_EQUAL;
2075 }
2076
2077 /* predicate = !predicate; */
2078 #define COMPARE_FALSE 1
2079 anv_batch_emit(batch, GENX(MI_PREDICATE), mip) {
2080 mip.LoadOperation = LOAD_LOADINV;
2081 mip.CombineOperation = COMBINE_OR;
2082 mip.CompareOperation = COMPARE_FALSE;
2083 }
2084 #endif
2085
2086 anv_batch_emit(batch, GENX(GPGPU_WALKER), ggw) {
2087 ggw.IndirectParameterEnable = true;
2088 ggw.PredicateEnable = GEN_GEN <= 7;
2089 ggw.SIMDSize = prog_data->simd_size / 16;
2090 ggw.ThreadDepthCounterMaximum = 0;
2091 ggw.ThreadHeightCounterMaximum = 0;
2092 ggw.ThreadWidthCounterMaximum = prog_data->threads - 1;
2093 ggw.RightExecutionMask = pipeline->cs_right_mask;
2094 ggw.BottomExecutionMask = 0xffffffff;
2095 }
2096
2097 anv_batch_emit(batch, GENX(MEDIA_STATE_FLUSH), msf);
2098 }
2099
2100 static void
2101 flush_pipeline_before_pipeline_select(struct anv_cmd_buffer *cmd_buffer,
2102 uint32_t pipeline)
2103 {
2104 #if GEN_GEN >= 8 && GEN_GEN < 10
2105 /* From the Broadwell PRM, Volume 2a: Instructions, PIPELINE_SELECT:
2106 *
2107 * Software must clear the COLOR_CALC_STATE Valid field in
2108 * 3DSTATE_CC_STATE_POINTERS command prior to send a PIPELINE_SELECT
2109 * with Pipeline Select set to GPGPU.
2110 *
2111 * The internal hardware docs recommend the same workaround for Gen9
2112 * hardware too.
2113 */
2114 if (pipeline == GPGPU)
2115 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CC_STATE_POINTERS), t);
2116 #elif GEN_GEN <= 7
2117 /* From "BXML » GT » MI » vol1a GPU Overview » [Instruction]
2118 * PIPELINE_SELECT [DevBWR+]":
2119 *
2120 * Project: DEVSNB+
2121 *
2122 * Software must ensure all the write caches are flushed through a
2123 * stalling PIPE_CONTROL command followed by another PIPE_CONTROL
2124 * command to invalidate read only caches prior to programming
2125 * MI_PIPELINE_SELECT command to change the Pipeline Select Mode.
2126 */
2127 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
2128 pc.RenderTargetCacheFlushEnable = true;
2129 pc.DepthCacheFlushEnable = true;
2130 pc.DCFlushEnable = true;
2131 pc.PostSyncOperation = NoWrite;
2132 pc.CommandStreamerStallEnable = true;
2133 }
2134
2135 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
2136 pc.TextureCacheInvalidationEnable = true;
2137 pc.ConstantCacheInvalidationEnable = true;
2138 pc.StateCacheInvalidationEnable = true;
2139 pc.InstructionCacheInvalidateEnable = true;
2140 pc.PostSyncOperation = NoWrite;
2141 }
2142 #endif
2143 }
2144
2145 void
2146 genX(flush_pipeline_select_3d)(struct anv_cmd_buffer *cmd_buffer)
2147 {
2148 if (cmd_buffer->state.current_pipeline != _3D) {
2149 flush_pipeline_before_pipeline_select(cmd_buffer, _3D);
2150
2151 anv_batch_emit(&cmd_buffer->batch, GENX(PIPELINE_SELECT), ps) {
2152 #if GEN_GEN >= 9
2153 ps.MaskBits = 3;
2154 #endif
2155 ps.PipelineSelection = _3D;
2156 }
2157
2158 cmd_buffer->state.current_pipeline = _3D;
2159 }
2160 }
2161
2162 void
2163 genX(flush_pipeline_select_gpgpu)(struct anv_cmd_buffer *cmd_buffer)
2164 {
2165 if (cmd_buffer->state.current_pipeline != GPGPU) {
2166 flush_pipeline_before_pipeline_select(cmd_buffer, GPGPU);
2167
2168 anv_batch_emit(&cmd_buffer->batch, GENX(PIPELINE_SELECT), ps) {
2169 #if GEN_GEN >= 9
2170 ps.MaskBits = 3;
2171 #endif
2172 ps.PipelineSelection = GPGPU;
2173 }
2174
2175 cmd_buffer->state.current_pipeline = GPGPU;
2176 }
2177 }
2178
2179 void
2180 genX(cmd_buffer_emit_gen7_depth_flush)(struct anv_cmd_buffer *cmd_buffer)
2181 {
2182 if (GEN_GEN >= 8)
2183 return;
2184
2185 /* From the Haswell PRM, documentation for 3DSTATE_DEPTH_BUFFER:
2186 *
2187 * "Restriction: Prior to changing Depth/Stencil Buffer state (i.e., any
2188 * combination of 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS,
2189 * 3DSTATE_STENCIL_BUFFER, 3DSTATE_HIER_DEPTH_BUFFER) SW must first
2190 * issue a pipelined depth stall (PIPE_CONTROL with Depth Stall bit
2191 * set), followed by a pipelined depth cache flush (PIPE_CONTROL with
2192 * Depth Flush Bit set, followed by another pipelined depth stall
2193 * (PIPE_CONTROL with Depth Stall Bit set), unless SW can otherwise
2194 * guarantee that the pipeline from WM onwards is already flushed (e.g.,
2195 * via a preceding MI_FLUSH)."
2196 */
2197 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pipe) {
2198 pipe.DepthStallEnable = true;
2199 }
2200 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pipe) {
2201 pipe.DepthCacheFlushEnable = true;
2202 }
2203 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pipe) {
2204 pipe.DepthStallEnable = true;
2205 }
2206 }
2207
2208 static uint32_t
2209 depth_stencil_surface_type(enum isl_surf_dim dim)
2210 {
2211 switch (dim) {
2212 case ISL_SURF_DIM_1D:
2213 if (GEN_GEN >= 9) {
2214 /* From the Sky Lake PRM, 3DSTATAE_DEPTH_BUFFER::SurfaceType
2215 *
2216 * Programming Notes:
2217 * The Surface Type of the depth buffer must be the same as the
2218 * Surface Type of the render target(s) (defined in
2219 * SURFACE_STATE), unless either the depth buffer or render
2220 * targets are SURFTYPE_NULL (see exception below for SKL). 1D
2221 * surface type not allowed for depth surface and stencil surface.
2222 *
2223 * Workaround:
2224 * If depth/stencil is enabled with 1D render target,
2225 * depth/stencil surface type needs to be set to 2D surface type
2226 * and height set to 1. Depth will use (legacy) TileY and stencil
2227 * will use TileW. For this case only, the Surface Type of the
2228 * depth buffer can be 2D while the Surface Type of the render
2229 * target(s) are 1D, representing an exception to a programming
2230 * note above.
2231 */
2232 return SURFTYPE_2D;
2233 } else {
2234 return SURFTYPE_1D;
2235 }
2236 case ISL_SURF_DIM_2D:
2237 return SURFTYPE_2D;
2238 case ISL_SURF_DIM_3D:
2239 if (GEN_GEN >= 9) {
2240 /* The Sky Lake docs list the value for 3D as "Reserved". However,
2241 * they have the exact same layout as 2D arrays on gen9+, so we can
2242 * just use 2D here.
2243 */
2244 return SURFTYPE_2D;
2245 } else {
2246 return SURFTYPE_3D;
2247 }
2248 default:
2249 unreachable("Invalid surface dimension");
2250 }
2251 }
2252
2253 static void
2254 cmd_buffer_emit_depth_stencil(struct anv_cmd_buffer *cmd_buffer)
2255 {
2256 struct anv_device *device = cmd_buffer->device;
2257 const struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
2258 const struct anv_image_view *iview =
2259 anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
2260 const struct anv_image *image = iview ? iview->image : NULL;
2261 const bool has_depth = image && (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT);
2262 const uint32_t ds = cmd_buffer->state.subpass->depth_stencil_attachment.attachment;
2263 const bool has_hiz = image != NULL &&
2264 cmd_buffer->state.attachments[ds].aux_usage == ISL_AUX_USAGE_HIZ;
2265 const bool has_stencil =
2266 image && (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT);
2267
2268 cmd_buffer->state.hiz_enabled = has_hiz;
2269
2270 /* FIXME: Width and Height are wrong */
2271
2272 genX(cmd_buffer_emit_gen7_depth_flush)(cmd_buffer);
2273
2274 /* Emit 3DSTATE_DEPTH_BUFFER */
2275 if (has_depth) {
2276 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER), db) {
2277 db.SurfaceType =
2278 depth_stencil_surface_type(image->depth_surface.isl.dim);
2279 db.DepthWriteEnable = true;
2280 db.StencilWriteEnable = has_stencil;
2281 db.HierarchicalDepthBufferEnable = has_hiz;
2282
2283 db.SurfaceFormat = isl_surf_get_depth_format(&device->isl_dev,
2284 &image->depth_surface.isl);
2285
2286 db.SurfaceBaseAddress = (struct anv_address) {
2287 .bo = image->bo,
2288 .offset = image->offset + image->depth_surface.offset,
2289 };
2290 db.DepthBufferObjectControlState = GENX(MOCS);
2291
2292 db.SurfacePitch = image->depth_surface.isl.row_pitch - 1;
2293 db.Height = image->extent.height - 1;
2294 db.Width = image->extent.width - 1;
2295 db.LOD = iview->isl.base_level;
2296 db.MinimumArrayElement = iview->isl.base_array_layer;
2297
2298 assert(image->depth_surface.isl.dim != ISL_SURF_DIM_3D);
2299 db.Depth =
2300 db.RenderTargetViewExtent = iview->isl.array_len - 1;
2301
2302 #if GEN_GEN >= 8
2303 db.SurfaceQPitch =
2304 isl_surf_get_array_pitch_el_rows(&image->depth_surface.isl) >> 2;
2305 #endif
2306 }
2307 } else {
2308 /* Even when no depth buffer is present, the hardware requires that
2309 * 3DSTATE_DEPTH_BUFFER be programmed correctly. The Broadwell PRM says:
2310 *
2311 * If a null depth buffer is bound, the driver must instead bind depth as:
2312 * 3DSTATE_DEPTH.SurfaceType = SURFTYPE_2D
2313 * 3DSTATE_DEPTH.Width = 1
2314 * 3DSTATE_DEPTH.Height = 1
2315 * 3DSTATE_DEPTH.SuraceFormat = D16_UNORM
2316 * 3DSTATE_DEPTH.SurfaceBaseAddress = 0
2317 * 3DSTATE_DEPTH.HierarchicalDepthBufferEnable = 0
2318 * 3DSTATE_WM_DEPTH_STENCIL.DepthTestEnable = 0
2319 * 3DSTATE_WM_DEPTH_STENCIL.DepthBufferWriteEnable = 0
2320 *
2321 * The PRM is wrong, though. The width and height must be programmed to
2322 * actual framebuffer's width and height, even when neither depth buffer
2323 * nor stencil buffer is present. Also, D16_UNORM is not allowed to
2324 * be combined with a stencil buffer so we use D32_FLOAT instead.
2325 */
2326 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER), db) {
2327 if (has_stencil) {
2328 db.SurfaceType =
2329 depth_stencil_surface_type(image->stencil_surface.isl.dim);
2330 } else {
2331 db.SurfaceType = SURFTYPE_2D;
2332 }
2333 db.SurfaceFormat = D32_FLOAT;
2334 db.Width = MAX2(fb->width, 1) - 1;
2335 db.Height = MAX2(fb->height, 1) - 1;
2336 db.StencilWriteEnable = has_stencil;
2337 }
2338 }
2339
2340 if (has_hiz) {
2341 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_HIER_DEPTH_BUFFER), hdb) {
2342 hdb.HierarchicalDepthBufferObjectControlState = GENX(MOCS);
2343 hdb.SurfacePitch = image->aux_surface.isl.row_pitch - 1;
2344 hdb.SurfaceBaseAddress = (struct anv_address) {
2345 .bo = image->bo,
2346 .offset = image->offset + image->aux_surface.offset,
2347 };
2348 #if GEN_GEN >= 8
2349 /* From the SKL PRM Vol2a:
2350 *
2351 * The interpretation of this field is dependent on Surface Type
2352 * as follows:
2353 * - SURFTYPE_1D: distance in pixels between array slices
2354 * - SURFTYPE_2D/CUBE: distance in rows between array slices
2355 * - SURFTYPE_3D: distance in rows between R - slices
2356 *
2357 * Unfortunately, the docs aren't 100% accurate here. They fail to
2358 * mention that the 1-D rule only applies to linear 1-D images.
2359 * Since depth and HiZ buffers are always tiled, they are treated as
2360 * 2-D images. Prior to Sky Lake, this field is always in rows.
2361 */
2362 hdb.SurfaceQPitch =
2363 isl_surf_get_array_pitch_sa_rows(&image->aux_surface.isl) >> 2;
2364 #endif
2365 }
2366 } else {
2367 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_HIER_DEPTH_BUFFER), hdb);
2368 }
2369
2370 /* Emit 3DSTATE_STENCIL_BUFFER */
2371 if (has_stencil) {
2372 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER), sb) {
2373 #if GEN_GEN >= 8 || GEN_IS_HASWELL
2374 sb.StencilBufferEnable = true;
2375 #endif
2376 sb.StencilBufferObjectControlState = GENX(MOCS);
2377
2378 sb.SurfacePitch = image->stencil_surface.isl.row_pitch - 1;
2379
2380 #if GEN_GEN >= 8
2381 sb.SurfaceQPitch = isl_surf_get_array_pitch_el_rows(&image->stencil_surface.isl) >> 2;
2382 #endif
2383 sb.SurfaceBaseAddress = (struct anv_address) {
2384 .bo = image->bo,
2385 .offset = image->offset + image->stencil_surface.offset,
2386 };
2387 }
2388 } else {
2389 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER), sb);
2390 }
2391
2392 /* From the IVB PRM Vol2P1, 11.5.5.4 3DSTATE_CLEAR_PARAMS:
2393 *
2394 * 3DSTATE_CLEAR_PARAMS must always be programmed in the along with
2395 * the other Depth/Stencil state commands(i.e. 3DSTATE_DEPTH_BUFFER,
2396 * 3DSTATE_STENCIL_BUFFER, or 3DSTATE_HIER_DEPTH_BUFFER)
2397 *
2398 * Testing also shows that some variant of this restriction may exist HSW+.
2399 * On BDW+, it is not possible to emit 2 of these packets consecutively when
2400 * both have DepthClearValueValid set. An analysis of such state programming
2401 * on SKL showed that the GPU doesn't register the latter packet's clear
2402 * value.
2403 */
2404 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CLEAR_PARAMS), cp) {
2405 if (has_hiz) {
2406 cp.DepthClearValueValid = true;
2407 cp.DepthClearValue = ANV_HZ_FC_VAL;
2408 }
2409 }
2410 }
2411
2412
2413 /**
2414 * @brief Perform any layout transitions required at the beginning and/or end
2415 * of the current subpass for depth buffers.
2416 *
2417 * TODO: Consider preprocessing the attachment reference array at render pass
2418 * create time to determine if no layout transition is needed at the
2419 * beginning and/or end of each subpass.
2420 *
2421 * @param cmd_buffer The command buffer the transition is happening within.
2422 * @param subpass_end If true, marks that the transition is happening at the
2423 * end of the subpass.
2424 */
2425 static void
2426 cmd_buffer_subpass_transition_layouts(struct anv_cmd_buffer * const cmd_buffer,
2427 const bool subpass_end)
2428 {
2429 /* We need a non-NULL command buffer. */
2430 assert(cmd_buffer);
2431
2432 const struct anv_cmd_state * const cmd_state = &cmd_buffer->state;
2433 const struct anv_subpass * const subpass = cmd_state->subpass;
2434
2435 /* This function must be called within a subpass. */
2436 assert(subpass);
2437
2438 /* If there are attachment references, the array shouldn't be NULL.
2439 */
2440 if (subpass->attachment_count > 0)
2441 assert(subpass->attachments);
2442
2443 /* Iterate over the array of attachment references. */
2444 for (const VkAttachmentReference *att_ref = subpass->attachments;
2445 att_ref < subpass->attachments + subpass->attachment_count; att_ref++) {
2446
2447 /* If the attachment is unused, we can't perform a layout transition. */
2448 if (att_ref->attachment == VK_ATTACHMENT_UNUSED)
2449 continue;
2450
2451 /* This attachment index shouldn't go out of bounds. */
2452 assert(att_ref->attachment < cmd_state->pass->attachment_count);
2453
2454 const struct anv_render_pass_attachment * const att_desc =
2455 &cmd_state->pass->attachments[att_ref->attachment];
2456 struct anv_attachment_state * const att_state =
2457 &cmd_buffer->state.attachments[att_ref->attachment];
2458
2459 /* The attachment should not be used in a subpass after its last. */
2460 assert(att_desc->last_subpass_idx >= anv_get_subpass_id(cmd_state));
2461
2462 if (subpass_end && anv_get_subpass_id(cmd_state) <
2463 att_desc->last_subpass_idx) {
2464 /* We're calling this function on a buffer twice in one subpass and
2465 * this is not the last use of the buffer. The layout should not have
2466 * changed from the first call and no transition is necessary.
2467 */
2468 assert(att_ref->layout == att_state->current_layout);
2469 continue;
2470 }
2471
2472 /* Get the appropriate target layout for this attachment. */
2473 const VkImageLayout target_layout = subpass_end ?
2474 att_desc->final_layout : att_ref->layout;
2475
2476 /* The attachment index must be less than the number of attachments
2477 * within the framebuffer.
2478 */
2479 assert(att_ref->attachment < cmd_state->framebuffer->attachment_count);
2480
2481 const struct anv_image * const image =
2482 cmd_state->framebuffer->attachments[att_ref->attachment]->image;
2483
2484 /* Perform the layout transition. */
2485 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
2486 transition_depth_buffer(cmd_buffer, image,
2487 att_state->current_layout, target_layout);
2488 att_state->aux_usage =
2489 anv_layout_to_aux_usage(&cmd_buffer->device->info, image,
2490 image->aspects, target_layout);
2491 }
2492
2493 att_state->current_layout = target_layout;
2494 }
2495 }
2496
2497 static void
2498 genX(cmd_buffer_set_subpass)(struct anv_cmd_buffer *cmd_buffer,
2499 struct anv_subpass *subpass)
2500 {
2501 cmd_buffer->state.subpass = subpass;
2502
2503 cmd_buffer->state.dirty |= ANV_CMD_DIRTY_RENDER_TARGETS;
2504
2505 /* Perform transitions to the subpass layout before any writes have
2506 * occurred.
2507 */
2508 cmd_buffer_subpass_transition_layouts(cmd_buffer, false);
2509
2510 cmd_buffer_emit_depth_stencil(cmd_buffer);
2511
2512 anv_cmd_buffer_clear_subpass(cmd_buffer);
2513 }
2514
2515 void genX(CmdBeginRenderPass)(
2516 VkCommandBuffer commandBuffer,
2517 const VkRenderPassBeginInfo* pRenderPassBegin,
2518 VkSubpassContents contents)
2519 {
2520 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
2521 ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
2522 ANV_FROM_HANDLE(anv_framebuffer, framebuffer, pRenderPassBegin->framebuffer);
2523
2524 cmd_buffer->state.framebuffer = framebuffer;
2525 cmd_buffer->state.pass = pass;
2526 cmd_buffer->state.render_area = pRenderPassBegin->renderArea;
2527 VkResult result =
2528 genX(cmd_buffer_setup_attachments)(cmd_buffer, pass, pRenderPassBegin);
2529
2530 /* If we failed to setup the attachments we should not try to go further */
2531 if (result != VK_SUCCESS) {
2532 assert(anv_batch_has_error(&cmd_buffer->batch));
2533 return;
2534 }
2535
2536 genX(flush_pipeline_select_3d)(cmd_buffer);
2537
2538 genX(cmd_buffer_set_subpass)(cmd_buffer, pass->subpasses);
2539 }
2540
2541 void genX(CmdNextSubpass)(
2542 VkCommandBuffer commandBuffer,
2543 VkSubpassContents contents)
2544 {
2545 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
2546
2547 if (anv_batch_has_error(&cmd_buffer->batch))
2548 return;
2549
2550 assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
2551
2552 anv_cmd_buffer_resolve_subpass(cmd_buffer);
2553
2554 /* Perform transitions to the final layout after all writes have occurred.
2555 */
2556 cmd_buffer_subpass_transition_layouts(cmd_buffer, true);
2557
2558 genX(cmd_buffer_set_subpass)(cmd_buffer, cmd_buffer->state.subpass + 1);
2559 }
2560
2561 void genX(CmdEndRenderPass)(
2562 VkCommandBuffer commandBuffer)
2563 {
2564 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
2565
2566 if (anv_batch_has_error(&cmd_buffer->batch))
2567 return;
2568
2569 anv_cmd_buffer_resolve_subpass(cmd_buffer);
2570
2571 /* Perform transitions to the final layout after all writes have occurred.
2572 */
2573 cmd_buffer_subpass_transition_layouts(cmd_buffer, true);
2574
2575 cmd_buffer->state.hiz_enabled = false;
2576
2577 #ifndef NDEBUG
2578 anv_dump_add_framebuffer(cmd_buffer, cmd_buffer->state.framebuffer);
2579 #endif
2580
2581 /* Remove references to render pass specific state. This enables us to
2582 * detect whether or not we're in a renderpass.
2583 */
2584 cmd_buffer->state.framebuffer = NULL;
2585 cmd_buffer->state.pass = NULL;
2586 cmd_buffer->state.subpass = NULL;
2587 }