a91cffd8fc7ae54892d574c555a238b7ce3fd1a6
[mesa.git] / src / intel / vulkan / genX_cmd_buffer.c
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26
27 #include "anv_private.h"
28 #include "vk_format_info.h"
29 #include "vk_util.h"
30 #include "util/fast_idiv_by_const.h"
31
32 #include "common/gen_l3_config.h"
33 #include "genxml/gen_macros.h"
34 #include "genxml/genX_pack.h"
35
36 /* We reserve GPR 14 and 15 for conditional rendering */
37 #define GEN_MI_BUILDER_NUM_ALLOC_GPRS 14
38 #define __gen_get_batch_dwords anv_batch_emit_dwords
39 #define __gen_address_offset anv_address_add
40 #include "common/gen_mi_builder.h"
41
42 static void
43 emit_lri(struct anv_batch *batch, uint32_t reg, uint32_t imm)
44 {
45 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
46 lri.RegisterOffset = reg;
47 lri.DataDWord = imm;
48 }
49 }
50
51 void
52 genX(cmd_buffer_emit_state_base_address)(struct anv_cmd_buffer *cmd_buffer)
53 {
54 struct anv_device *device = cmd_buffer->device;
55
56 /* If we are emitting a new state base address we probably need to re-emit
57 * binding tables.
58 */
59 cmd_buffer->state.descriptors_dirty |= ~0;
60
61 /* Emit a render target cache flush.
62 *
63 * This isn't documented anywhere in the PRM. However, it seems to be
64 * necessary prior to changing the surface state base adress. Without
65 * this, we get GPU hangs when using multi-level command buffers which
66 * clear depth, reset state base address, and then go render stuff.
67 */
68 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
69 pc.DCFlushEnable = true;
70 pc.RenderTargetCacheFlushEnable = true;
71 pc.CommandStreamerStallEnable = true;
72 }
73
74 anv_batch_emit(&cmd_buffer->batch, GENX(STATE_BASE_ADDRESS), sba) {
75 sba.GeneralStateBaseAddress = (struct anv_address) { NULL, 0 };
76 sba.GeneralStateMOCS = GENX(MOCS);
77 sba.GeneralStateBaseAddressModifyEnable = true;
78
79 sba.StatelessDataPortAccessMOCS = GENX(MOCS);
80
81 sba.SurfaceStateBaseAddress =
82 anv_cmd_buffer_surface_base_address(cmd_buffer);
83 sba.SurfaceStateMOCS = GENX(MOCS);
84 sba.SurfaceStateBaseAddressModifyEnable = true;
85
86 sba.DynamicStateBaseAddress =
87 (struct anv_address) { device->dynamic_state_pool.block_pool.bo, 0 };
88 sba.DynamicStateMOCS = GENX(MOCS);
89 sba.DynamicStateBaseAddressModifyEnable = true;
90
91 sba.IndirectObjectBaseAddress = (struct anv_address) { NULL, 0 };
92 sba.IndirectObjectMOCS = GENX(MOCS);
93 sba.IndirectObjectBaseAddressModifyEnable = true;
94
95 sba.InstructionBaseAddress =
96 (struct anv_address) { device->instruction_state_pool.block_pool.bo, 0 };
97 sba.InstructionMOCS = GENX(MOCS);
98 sba.InstructionBaseAddressModifyEnable = true;
99
100 # if (GEN_GEN >= 8)
101 /* Broadwell requires that we specify a buffer size for a bunch of
102 * these fields. However, since we will be growing the BO's live, we
103 * just set them all to the maximum.
104 */
105 sba.GeneralStateBufferSize = 0xfffff;
106 sba.GeneralStateBufferSizeModifyEnable = true;
107 sba.DynamicStateBufferSize = 0xfffff;
108 sba.DynamicStateBufferSizeModifyEnable = true;
109 sba.IndirectObjectBufferSize = 0xfffff;
110 sba.IndirectObjectBufferSizeModifyEnable = true;
111 sba.InstructionBufferSize = 0xfffff;
112 sba.InstructionBuffersizeModifyEnable = true;
113 # else
114 /* On gen7, we have upper bounds instead. According to the docs,
115 * setting an upper bound of zero means that no bounds checking is
116 * performed so, in theory, we should be able to leave them zero.
117 * However, border color is broken and the GPU bounds-checks anyway.
118 * To avoid this and other potential problems, we may as well set it
119 * for everything.
120 */
121 sba.GeneralStateAccessUpperBound =
122 (struct anv_address) { .bo = NULL, .offset = 0xfffff000 };
123 sba.GeneralStateAccessUpperBoundModifyEnable = true;
124 sba.DynamicStateAccessUpperBound =
125 (struct anv_address) { .bo = NULL, .offset = 0xfffff000 };
126 sba.DynamicStateAccessUpperBoundModifyEnable = true;
127 sba.InstructionAccessUpperBound =
128 (struct anv_address) { .bo = NULL, .offset = 0xfffff000 };
129 sba.InstructionAccessUpperBoundModifyEnable = true;
130 # endif
131 # if (GEN_GEN >= 9)
132 if (cmd_buffer->device->instance->physicalDevice.use_softpin) {
133 sba.BindlessSurfaceStateBaseAddress = (struct anv_address) {
134 .bo = device->surface_state_pool.block_pool.bo,
135 .offset = 0,
136 };
137 sba.BindlessSurfaceStateSize = (1 << 20) - 1;
138 } else {
139 sba.BindlessSurfaceStateBaseAddress = ANV_NULL_ADDRESS;
140 sba.BindlessSurfaceStateSize = 0;
141 }
142 sba.BindlessSurfaceStateMOCS = GENX(MOCS);
143 sba.BindlessSurfaceStateBaseAddressModifyEnable = true;
144 # endif
145 # if (GEN_GEN >= 10)
146 sba.BindlessSamplerStateBaseAddress = (struct anv_address) { NULL, 0 };
147 sba.BindlessSamplerStateMOCS = GENX(MOCS);
148 sba.BindlessSamplerStateBaseAddressModifyEnable = true;
149 sba.BindlessSamplerStateBufferSize = 0;
150 # endif
151 }
152
153 /* After re-setting the surface state base address, we have to do some
154 * cache flusing so that the sampler engine will pick up the new
155 * SURFACE_STATE objects and binding tables. From the Broadwell PRM,
156 * Shared Function > 3D Sampler > State > State Caching (page 96):
157 *
158 * Coherency with system memory in the state cache, like the texture
159 * cache is handled partially by software. It is expected that the
160 * command stream or shader will issue Cache Flush operation or
161 * Cache_Flush sampler message to ensure that the L1 cache remains
162 * coherent with system memory.
163 *
164 * [...]
165 *
166 * Whenever the value of the Dynamic_State_Base_Addr,
167 * Surface_State_Base_Addr are altered, the L1 state cache must be
168 * invalidated to ensure the new surface or sampler state is fetched
169 * from system memory.
170 *
171 * The PIPE_CONTROL command has a "State Cache Invalidation Enable" bit
172 * which, according the PIPE_CONTROL instruction documentation in the
173 * Broadwell PRM:
174 *
175 * Setting this bit is independent of any other bit in this packet.
176 * This bit controls the invalidation of the L1 and L2 state caches
177 * at the top of the pipe i.e. at the parsing time.
178 *
179 * Unfortunately, experimentation seems to indicate that state cache
180 * invalidation through a PIPE_CONTROL does nothing whatsoever in
181 * regards to surface state and binding tables. In stead, it seems that
182 * invalidating the texture cache is what is actually needed.
183 *
184 * XXX: As far as we have been able to determine through
185 * experimentation, shows that flush the texture cache appears to be
186 * sufficient. The theory here is that all of the sampling/rendering
187 * units cache the binding table in the texture cache. However, we have
188 * yet to be able to actually confirm this.
189 */
190 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
191 pc.TextureCacheInvalidationEnable = true;
192 pc.ConstantCacheInvalidationEnable = true;
193 pc.StateCacheInvalidationEnable = true;
194 }
195 }
196
197 static void
198 add_surface_reloc(struct anv_cmd_buffer *cmd_buffer,
199 struct anv_state state, struct anv_address addr)
200 {
201 const struct isl_device *isl_dev = &cmd_buffer->device->isl_dev;
202
203 VkResult result =
204 anv_reloc_list_add(&cmd_buffer->surface_relocs, &cmd_buffer->pool->alloc,
205 state.offset + isl_dev->ss.addr_offset,
206 addr.bo, addr.offset);
207 if (result != VK_SUCCESS)
208 anv_batch_set_error(&cmd_buffer->batch, result);
209 }
210
211 static void
212 add_surface_state_relocs(struct anv_cmd_buffer *cmd_buffer,
213 struct anv_surface_state state)
214 {
215 const struct isl_device *isl_dev = &cmd_buffer->device->isl_dev;
216
217 assert(!anv_address_is_null(state.address));
218 add_surface_reloc(cmd_buffer, state.state, state.address);
219
220 if (!anv_address_is_null(state.aux_address)) {
221 VkResult result =
222 anv_reloc_list_add(&cmd_buffer->surface_relocs,
223 &cmd_buffer->pool->alloc,
224 state.state.offset + isl_dev->ss.aux_addr_offset,
225 state.aux_address.bo, state.aux_address.offset);
226 if (result != VK_SUCCESS)
227 anv_batch_set_error(&cmd_buffer->batch, result);
228 }
229
230 if (!anv_address_is_null(state.clear_address)) {
231 VkResult result =
232 anv_reloc_list_add(&cmd_buffer->surface_relocs,
233 &cmd_buffer->pool->alloc,
234 state.state.offset +
235 isl_dev->ss.clear_color_state_offset,
236 state.clear_address.bo, state.clear_address.offset);
237 if (result != VK_SUCCESS)
238 anv_batch_set_error(&cmd_buffer->batch, result);
239 }
240 }
241
242 static void
243 color_attachment_compute_aux_usage(struct anv_device * device,
244 struct anv_cmd_state * cmd_state,
245 uint32_t att, VkRect2D render_area,
246 union isl_color_value *fast_clear_color)
247 {
248 struct anv_attachment_state *att_state = &cmd_state->attachments[att];
249 struct anv_image_view *iview = cmd_state->attachments[att].image_view;
250
251 assert(iview->n_planes == 1);
252
253 if (iview->planes[0].isl.base_array_layer >=
254 anv_image_aux_layers(iview->image, VK_IMAGE_ASPECT_COLOR_BIT,
255 iview->planes[0].isl.base_level)) {
256 /* There is no aux buffer which corresponds to the level and layer(s)
257 * being accessed.
258 */
259 att_state->aux_usage = ISL_AUX_USAGE_NONE;
260 att_state->input_aux_usage = ISL_AUX_USAGE_NONE;
261 att_state->fast_clear = false;
262 return;
263 }
264
265 att_state->aux_usage =
266 anv_layout_to_aux_usage(&device->info, iview->image,
267 VK_IMAGE_ASPECT_COLOR_BIT,
268 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
269
270 /* If we don't have aux, then we should have returned early in the layer
271 * check above. If we got here, we must have something.
272 */
273 assert(att_state->aux_usage != ISL_AUX_USAGE_NONE);
274
275 if (att_state->aux_usage == ISL_AUX_USAGE_CCS_E ||
276 att_state->aux_usage == ISL_AUX_USAGE_MCS) {
277 att_state->input_aux_usage = att_state->aux_usage;
278 } else {
279 /* From the Sky Lake PRM, RENDER_SURFACE_STATE::AuxiliarySurfaceMode:
280 *
281 * "If Number of Multisamples is MULTISAMPLECOUNT_1, AUX_CCS_D
282 * setting is only allowed if Surface Format supported for Fast
283 * Clear. In addition, if the surface is bound to the sampling
284 * engine, Surface Format must be supported for Render Target
285 * Compression for surfaces bound to the sampling engine."
286 *
287 * In other words, we can only sample from a fast-cleared image if it
288 * also supports color compression.
289 */
290 if (isl_format_supports_ccs_e(&device->info, iview->planes[0].isl.format)) {
291 att_state->input_aux_usage = ISL_AUX_USAGE_CCS_D;
292
293 /* While fast-clear resolves and partial resolves are fairly cheap in the
294 * case where you render to most of the pixels, full resolves are not
295 * because they potentially involve reading and writing the entire
296 * framebuffer. If we can't texture with CCS_E, we should leave it off and
297 * limit ourselves to fast clears.
298 */
299 if (cmd_state->pass->attachments[att].first_subpass_layout ==
300 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL) {
301 anv_perf_warn(device->instance, iview->image,
302 "Not temporarily enabling CCS_E.");
303 }
304 } else {
305 att_state->input_aux_usage = ISL_AUX_USAGE_NONE;
306 }
307 }
308
309 assert(iview->image->planes[0].aux_surface.isl.usage &
310 (ISL_SURF_USAGE_CCS_BIT | ISL_SURF_USAGE_MCS_BIT));
311
312 union isl_color_value clear_color = {};
313 anv_clear_color_from_att_state(&clear_color, att_state, iview);
314
315 att_state->clear_color_is_zero_one =
316 isl_color_value_is_zero_one(clear_color, iview->planes[0].isl.format);
317 att_state->clear_color_is_zero =
318 isl_color_value_is_zero(clear_color, iview->planes[0].isl.format);
319
320 if (att_state->pending_clear_aspects == VK_IMAGE_ASPECT_COLOR_BIT) {
321 /* Start by getting the fast clear type. We use the first subpass
322 * layout here because we don't want to fast-clear if the first subpass
323 * to use the attachment can't handle fast-clears.
324 */
325 enum anv_fast_clear_type fast_clear_type =
326 anv_layout_to_fast_clear_type(&device->info, iview->image,
327 VK_IMAGE_ASPECT_COLOR_BIT,
328 cmd_state->pass->attachments[att].first_subpass_layout);
329 switch (fast_clear_type) {
330 case ANV_FAST_CLEAR_NONE:
331 att_state->fast_clear = false;
332 break;
333 case ANV_FAST_CLEAR_DEFAULT_VALUE:
334 att_state->fast_clear = att_state->clear_color_is_zero;
335 break;
336 case ANV_FAST_CLEAR_ANY:
337 att_state->fast_clear = true;
338 break;
339 }
340
341 /* Potentially, we could do partial fast-clears but doing so has crazy
342 * alignment restrictions. It's easier to just restrict to full size
343 * fast clears for now.
344 */
345 if (render_area.offset.x != 0 ||
346 render_area.offset.y != 0 ||
347 render_area.extent.width != iview->extent.width ||
348 render_area.extent.height != iview->extent.height)
349 att_state->fast_clear = false;
350
351 /* On Broadwell and earlier, we can only handle 0/1 clear colors */
352 if (GEN_GEN <= 8 && !att_state->clear_color_is_zero_one)
353 att_state->fast_clear = false;
354
355 /* We only allow fast clears to the first slice of an image (level 0,
356 * layer 0) and only for the entire slice. This guarantees us that, at
357 * any given time, there is only one clear color on any given image at
358 * any given time. At the time of our testing (Jan 17, 2018), there
359 * were no known applications which would benefit from fast-clearing
360 * more than just the first slice.
361 */
362 if (att_state->fast_clear &&
363 (iview->planes[0].isl.base_level > 0 ||
364 iview->planes[0].isl.base_array_layer > 0)) {
365 anv_perf_warn(device->instance, iview->image,
366 "Rendering with multi-lod or multi-layer framebuffer "
367 "with LOAD_OP_LOAD and baseMipLevel > 0 or "
368 "baseArrayLayer > 0. Not fast clearing.");
369 att_state->fast_clear = false;
370 } else if (att_state->fast_clear && cmd_state->framebuffer->layers > 1) {
371 anv_perf_warn(device->instance, iview->image,
372 "Rendering to a multi-layer framebuffer with "
373 "LOAD_OP_CLEAR. Only fast-clearing the first slice");
374 }
375
376 if (att_state->fast_clear)
377 *fast_clear_color = clear_color;
378 } else {
379 att_state->fast_clear = false;
380 }
381 }
382
383 static void
384 depth_stencil_attachment_compute_aux_usage(struct anv_device *device,
385 struct anv_cmd_state *cmd_state,
386 uint32_t att, VkRect2D render_area)
387 {
388 struct anv_render_pass_attachment *pass_att =
389 &cmd_state->pass->attachments[att];
390 struct anv_attachment_state *att_state = &cmd_state->attachments[att];
391 struct anv_image_view *iview = cmd_state->attachments[att].image_view;
392
393 /* These will be initialized after the first subpass transition. */
394 att_state->aux_usage = ISL_AUX_USAGE_NONE;
395 att_state->input_aux_usage = ISL_AUX_USAGE_NONE;
396
397 if (GEN_GEN == 7) {
398 /* We don't do any HiZ or depth fast-clears on gen7 yet */
399 att_state->fast_clear = false;
400 return;
401 }
402
403 if (!(att_state->pending_clear_aspects & VK_IMAGE_ASPECT_DEPTH_BIT)) {
404 /* If we're just clearing stencil, we can always HiZ clear */
405 att_state->fast_clear = true;
406 return;
407 }
408
409 /* Default to false for now */
410 att_state->fast_clear = false;
411
412 /* We must have depth in order to have HiZ */
413 if (!(iview->image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT))
414 return;
415
416 const enum isl_aux_usage first_subpass_aux_usage =
417 anv_layout_to_aux_usage(&device->info, iview->image,
418 VK_IMAGE_ASPECT_DEPTH_BIT,
419 pass_att->first_subpass_layout);
420 if (first_subpass_aux_usage != ISL_AUX_USAGE_HIZ)
421 return;
422
423 if (!blorp_can_hiz_clear_depth(GEN_GEN,
424 iview->planes[0].isl.format,
425 iview->image->samples,
426 render_area.offset.x,
427 render_area.offset.y,
428 render_area.offset.x +
429 render_area.extent.width,
430 render_area.offset.y +
431 render_area.extent.height))
432 return;
433
434 if (att_state->clear_value.depthStencil.depth != ANV_HZ_FC_VAL)
435 return;
436
437 if (GEN_GEN == 8 && anv_can_sample_with_hiz(&device->info, iview->image)) {
438 /* Only gen9+ supports returning ANV_HZ_FC_VAL when sampling a
439 * fast-cleared portion of a HiZ buffer. Testing has revealed that Gen8
440 * only supports returning 0.0f. Gens prior to gen8 do not support this
441 * feature at all.
442 */
443 return;
444 }
445
446 /* If we got here, then we can fast clear */
447 att_state->fast_clear = true;
448 }
449
450 static bool
451 need_input_attachment_state(const struct anv_render_pass_attachment *att)
452 {
453 if (!(att->usage & VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT))
454 return false;
455
456 /* We only allocate input attachment states for color surfaces. Compression
457 * is not yet enabled for depth textures and stencil doesn't allow
458 * compression so we can just use the texture surface state from the view.
459 */
460 return vk_format_is_color(att->format);
461 }
462
463 /* Transitions a HiZ-enabled depth buffer from one layout to another. Unless
464 * the initial layout is undefined, the HiZ buffer and depth buffer will
465 * represent the same data at the end of this operation.
466 */
467 static void
468 transition_depth_buffer(struct anv_cmd_buffer *cmd_buffer,
469 const struct anv_image *image,
470 VkImageLayout initial_layout,
471 VkImageLayout final_layout)
472 {
473 const bool hiz_enabled = ISL_AUX_USAGE_HIZ ==
474 anv_layout_to_aux_usage(&cmd_buffer->device->info, image,
475 VK_IMAGE_ASPECT_DEPTH_BIT, initial_layout);
476 const bool enable_hiz = ISL_AUX_USAGE_HIZ ==
477 anv_layout_to_aux_usage(&cmd_buffer->device->info, image,
478 VK_IMAGE_ASPECT_DEPTH_BIT, final_layout);
479
480 enum isl_aux_op hiz_op;
481 if (hiz_enabled && !enable_hiz) {
482 hiz_op = ISL_AUX_OP_FULL_RESOLVE;
483 } else if (!hiz_enabled && enable_hiz) {
484 hiz_op = ISL_AUX_OP_AMBIGUATE;
485 } else {
486 assert(hiz_enabled == enable_hiz);
487 /* If the same buffer will be used, no resolves are necessary. */
488 hiz_op = ISL_AUX_OP_NONE;
489 }
490
491 if (hiz_op != ISL_AUX_OP_NONE)
492 anv_image_hiz_op(cmd_buffer, image, VK_IMAGE_ASPECT_DEPTH_BIT,
493 0, 0, 1, hiz_op);
494 }
495
496 static inline bool
497 vk_image_layout_stencil_write_optimal(VkImageLayout layout)
498 {
499 return layout == VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL ||
500 layout == VK_IMAGE_LAYOUT_DEPTH_READ_ONLY_STENCIL_ATTACHMENT_OPTIMAL;
501 }
502
503 /* Transitions a HiZ-enabled depth buffer from one layout to another. Unless
504 * the initial layout is undefined, the HiZ buffer and depth buffer will
505 * represent the same data at the end of this operation.
506 */
507 static void
508 transition_stencil_buffer(struct anv_cmd_buffer *cmd_buffer,
509 const struct anv_image *image,
510 uint32_t base_level, uint32_t level_count,
511 uint32_t base_layer, uint32_t layer_count,
512 VkImageLayout initial_layout,
513 VkImageLayout final_layout)
514 {
515 #if GEN_GEN == 7
516 uint32_t plane = anv_image_aspect_to_plane(image->aspects,
517 VK_IMAGE_ASPECT_STENCIL_BIT);
518
519 /* On gen7, we have to store a texturable version of the stencil buffer in
520 * a shadow whenever VK_IMAGE_USAGE_SAMPLED_BIT is set and copy back and
521 * forth at strategic points. Stencil writes are only allowed in three
522 * layouts:
523 *
524 * - VK_IMAGE_LAYOUT_GENERAL
525 * - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
526 * - VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL
527 * - VK_IMAGE_LAYOUT_DEPTH_READ_ONLY_STENCIL_ATTACHMENT_OPTIMAL
528 *
529 * For general, we have no nice opportunity to transition so we do the copy
530 * to the shadow unconditionally at the end of the subpass. For transfer
531 * destinations, we can update it as part of the transfer op. For the
532 * other two, we delay the copy until a transition into some other layout.
533 */
534 if (image->planes[plane].shadow_surface.isl.size_B > 0 &&
535 vk_image_layout_stencil_write_optimal(initial_layout) &&
536 !vk_image_layout_stencil_write_optimal(final_layout)) {
537 anv_image_copy_to_shadow(cmd_buffer, image,
538 VK_IMAGE_ASPECT_STENCIL_BIT,
539 base_level, level_count,
540 base_layer, layer_count);
541 }
542 #endif /* GEN_GEN == 7 */
543 }
544
545 #define MI_PREDICATE_SRC0 0x2400
546 #define MI_PREDICATE_SRC1 0x2408
547 #define MI_PREDICATE_RESULT 0x2418
548
549 static void
550 set_image_compressed_bit(struct anv_cmd_buffer *cmd_buffer,
551 const struct anv_image *image,
552 VkImageAspectFlagBits aspect,
553 uint32_t level,
554 uint32_t base_layer, uint32_t layer_count,
555 bool compressed)
556 {
557 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
558
559 /* We only have compression tracking for CCS_E */
560 if (image->planes[plane].aux_usage != ISL_AUX_USAGE_CCS_E)
561 return;
562
563 for (uint32_t a = 0; a < layer_count; a++) {
564 uint32_t layer = base_layer + a;
565 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_DATA_IMM), sdi) {
566 sdi.Address = anv_image_get_compression_state_addr(cmd_buffer->device,
567 image, aspect,
568 level, layer);
569 sdi.ImmediateData = compressed ? UINT32_MAX : 0;
570 }
571 }
572 }
573
574 static void
575 set_image_fast_clear_state(struct anv_cmd_buffer *cmd_buffer,
576 const struct anv_image *image,
577 VkImageAspectFlagBits aspect,
578 enum anv_fast_clear_type fast_clear)
579 {
580 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_DATA_IMM), sdi) {
581 sdi.Address = anv_image_get_fast_clear_type_addr(cmd_buffer->device,
582 image, aspect);
583 sdi.ImmediateData = fast_clear;
584 }
585
586 /* Whenever we have fast-clear, we consider that slice to be compressed.
587 * This makes building predicates much easier.
588 */
589 if (fast_clear != ANV_FAST_CLEAR_NONE)
590 set_image_compressed_bit(cmd_buffer, image, aspect, 0, 0, 1, true);
591 }
592
593 #if GEN_IS_HASWELL || GEN_GEN >= 8
594 static inline uint32_t
595 mi_alu(uint32_t opcode, uint32_t operand1, uint32_t operand2)
596 {
597 struct GENX(MI_MATH_ALU_INSTRUCTION) instr = {
598 .ALUOpcode = opcode,
599 .Operand1 = operand1,
600 .Operand2 = operand2,
601 };
602
603 uint32_t dw;
604 GENX(MI_MATH_ALU_INSTRUCTION_pack)(NULL, &dw, &instr);
605
606 return dw;
607 }
608 #endif
609
610 /* This is only really practical on haswell and above because it requires
611 * MI math in order to get it correct.
612 */
613 #if GEN_GEN >= 8 || GEN_IS_HASWELL
614 static void
615 anv_cmd_compute_resolve_predicate(struct anv_cmd_buffer *cmd_buffer,
616 const struct anv_image *image,
617 VkImageAspectFlagBits aspect,
618 uint32_t level, uint32_t array_layer,
619 enum isl_aux_op resolve_op,
620 enum anv_fast_clear_type fast_clear_supported)
621 {
622 struct gen_mi_builder b;
623 gen_mi_builder_init(&b, &cmd_buffer->batch);
624
625 const struct gen_mi_value fast_clear_type =
626 gen_mi_mem32(anv_image_get_fast_clear_type_addr(cmd_buffer->device,
627 image, aspect));
628
629 if (resolve_op == ISL_AUX_OP_FULL_RESOLVE) {
630 /* In this case, we're doing a full resolve which means we want the
631 * resolve to happen if any compression (including fast-clears) is
632 * present.
633 *
634 * In order to simplify the logic a bit, we make the assumption that,
635 * if the first slice has been fast-cleared, it is also marked as
636 * compressed. See also set_image_fast_clear_state.
637 */
638 const struct gen_mi_value compression_state =
639 gen_mi_mem32(anv_image_get_compression_state_addr(cmd_buffer->device,
640 image, aspect,
641 level, array_layer));
642 gen_mi_store(&b, gen_mi_reg64(MI_PREDICATE_SRC0),
643 compression_state);
644 gen_mi_store(&b, compression_state, gen_mi_imm(0));
645
646 if (level == 0 && array_layer == 0) {
647 /* If the predicate is true, we want to write 0 to the fast clear type
648 * and, if it's false, leave it alone. We can do this by writing
649 *
650 * clear_type = clear_type & ~predicate;
651 */
652 struct gen_mi_value new_fast_clear_type =
653 gen_mi_iand(&b, fast_clear_type,
654 gen_mi_inot(&b, gen_mi_reg64(MI_PREDICATE_SRC0)));
655 gen_mi_store(&b, fast_clear_type, new_fast_clear_type);
656 }
657 } else if (level == 0 && array_layer == 0) {
658 /* In this case, we are doing a partial resolve to get rid of fast-clear
659 * colors. We don't care about the compression state but we do care
660 * about how much fast clear is allowed by the final layout.
661 */
662 assert(resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE);
663 assert(fast_clear_supported < ANV_FAST_CLEAR_ANY);
664
665 /* We need to compute (fast_clear_supported < image->fast_clear) */
666 struct gen_mi_value pred =
667 gen_mi_ult(&b, gen_mi_imm(fast_clear_supported), fast_clear_type);
668 gen_mi_store(&b, gen_mi_reg64(MI_PREDICATE_SRC0),
669 gen_mi_value_ref(&b, pred));
670
671 /* If the predicate is true, we want to write 0 to the fast clear type
672 * and, if it's false, leave it alone. We can do this by writing
673 *
674 * clear_type = clear_type & ~predicate;
675 */
676 struct gen_mi_value new_fast_clear_type =
677 gen_mi_iand(&b, fast_clear_type, gen_mi_inot(&b, pred));
678 gen_mi_store(&b, fast_clear_type, new_fast_clear_type);
679 } else {
680 /* In this case, we're trying to do a partial resolve on a slice that
681 * doesn't have clear color. There's nothing to do.
682 */
683 assert(resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE);
684 return;
685 }
686
687 /* Set src1 to 0 and use a != condition */
688 gen_mi_store(&b, gen_mi_reg64(MI_PREDICATE_SRC1), gen_mi_imm(0));
689
690 anv_batch_emit(&cmd_buffer->batch, GENX(MI_PREDICATE), mip) {
691 mip.LoadOperation = LOAD_LOADINV;
692 mip.CombineOperation = COMBINE_SET;
693 mip.CompareOperation = COMPARE_SRCS_EQUAL;
694 }
695 }
696 #endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */
697
698 #if GEN_GEN <= 8
699 static void
700 anv_cmd_simple_resolve_predicate(struct anv_cmd_buffer *cmd_buffer,
701 const struct anv_image *image,
702 VkImageAspectFlagBits aspect,
703 uint32_t level, uint32_t array_layer,
704 enum isl_aux_op resolve_op,
705 enum anv_fast_clear_type fast_clear_supported)
706 {
707 struct gen_mi_builder b;
708 gen_mi_builder_init(&b, &cmd_buffer->batch);
709
710 struct gen_mi_value fast_clear_type_mem =
711 gen_mi_mem32(anv_image_get_fast_clear_type_addr(cmd_buffer->device,
712 image, aspect));
713
714 /* This only works for partial resolves and only when the clear color is
715 * all or nothing. On the upside, this emits less command streamer code
716 * and works on Ivybridge and Bay Trail.
717 */
718 assert(resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE);
719 assert(fast_clear_supported != ANV_FAST_CLEAR_ANY);
720
721 /* We don't support fast clears on anything other than the first slice. */
722 if (level > 0 || array_layer > 0)
723 return;
724
725 /* On gen8, we don't have a concept of default clear colors because we
726 * can't sample from CCS surfaces. It's enough to just load the fast clear
727 * state into the predicate register.
728 */
729 gen_mi_store(&b, gen_mi_reg64(MI_PREDICATE_SRC0), fast_clear_type_mem);
730 gen_mi_store(&b, gen_mi_reg64(MI_PREDICATE_SRC1), gen_mi_imm(0));
731 gen_mi_store(&b, fast_clear_type_mem, gen_mi_imm(0));
732
733 anv_batch_emit(&cmd_buffer->batch, GENX(MI_PREDICATE), mip) {
734 mip.LoadOperation = LOAD_LOADINV;
735 mip.CombineOperation = COMBINE_SET;
736 mip.CompareOperation = COMPARE_SRCS_EQUAL;
737 }
738 }
739 #endif /* GEN_GEN <= 8 */
740
741 static void
742 anv_cmd_predicated_ccs_resolve(struct anv_cmd_buffer *cmd_buffer,
743 const struct anv_image *image,
744 enum isl_format format,
745 VkImageAspectFlagBits aspect,
746 uint32_t level, uint32_t array_layer,
747 enum isl_aux_op resolve_op,
748 enum anv_fast_clear_type fast_clear_supported)
749 {
750 const uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
751
752 #if GEN_GEN >= 9
753 anv_cmd_compute_resolve_predicate(cmd_buffer, image,
754 aspect, level, array_layer,
755 resolve_op, fast_clear_supported);
756 #else /* GEN_GEN <= 8 */
757 anv_cmd_simple_resolve_predicate(cmd_buffer, image,
758 aspect, level, array_layer,
759 resolve_op, fast_clear_supported);
760 #endif
761
762 /* CCS_D only supports full resolves and BLORP will assert on us if we try
763 * to do a partial resolve on a CCS_D surface.
764 */
765 if (resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE &&
766 image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE)
767 resolve_op = ISL_AUX_OP_FULL_RESOLVE;
768
769 anv_image_ccs_op(cmd_buffer, image, format, aspect, level,
770 array_layer, 1, resolve_op, NULL, true);
771 }
772
773 static void
774 anv_cmd_predicated_mcs_resolve(struct anv_cmd_buffer *cmd_buffer,
775 const struct anv_image *image,
776 enum isl_format format,
777 VkImageAspectFlagBits aspect,
778 uint32_t array_layer,
779 enum isl_aux_op resolve_op,
780 enum anv_fast_clear_type fast_clear_supported)
781 {
782 assert(aspect == VK_IMAGE_ASPECT_COLOR_BIT);
783 assert(resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE);
784
785 #if GEN_GEN >= 8 || GEN_IS_HASWELL
786 anv_cmd_compute_resolve_predicate(cmd_buffer, image,
787 aspect, 0, array_layer,
788 resolve_op, fast_clear_supported);
789
790 anv_image_mcs_op(cmd_buffer, image, format, aspect,
791 array_layer, 1, resolve_op, NULL, true);
792 #else
793 unreachable("MCS resolves are unsupported on Ivybridge and Bay Trail");
794 #endif
795 }
796
797 void
798 genX(cmd_buffer_mark_image_written)(struct anv_cmd_buffer *cmd_buffer,
799 const struct anv_image *image,
800 VkImageAspectFlagBits aspect,
801 enum isl_aux_usage aux_usage,
802 uint32_t level,
803 uint32_t base_layer,
804 uint32_t layer_count)
805 {
806 /* The aspect must be exactly one of the image aspects. */
807 assert(util_bitcount(aspect) == 1 && (aspect & image->aspects));
808
809 /* The only compression types with more than just fast-clears are MCS,
810 * CCS_E, and HiZ. With HiZ we just trust the layout and don't actually
811 * track the current fast-clear and compression state. This leaves us
812 * with just MCS and CCS_E.
813 */
814 if (aux_usage != ISL_AUX_USAGE_CCS_E &&
815 aux_usage != ISL_AUX_USAGE_MCS)
816 return;
817
818 set_image_compressed_bit(cmd_buffer, image, aspect,
819 level, base_layer, layer_count, true);
820 }
821
822 static void
823 init_fast_clear_color(struct anv_cmd_buffer *cmd_buffer,
824 const struct anv_image *image,
825 VkImageAspectFlagBits aspect)
826 {
827 assert(cmd_buffer && image);
828 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
829
830 set_image_fast_clear_state(cmd_buffer, image, aspect,
831 ANV_FAST_CLEAR_NONE);
832
833 /* Initialize the struct fields that are accessed for fast-clears so that
834 * the HW restrictions on the field values are satisfied.
835 */
836 struct anv_address addr =
837 anv_image_get_clear_color_addr(cmd_buffer->device, image, aspect);
838
839 if (GEN_GEN >= 9) {
840 const struct isl_device *isl_dev = &cmd_buffer->device->isl_dev;
841 const unsigned num_dwords = GEN_GEN >= 10 ?
842 isl_dev->ss.clear_color_state_size / 4 :
843 isl_dev->ss.clear_value_size / 4;
844 for (unsigned i = 0; i < num_dwords; i++) {
845 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_DATA_IMM), sdi) {
846 sdi.Address = addr;
847 sdi.Address.offset += i * 4;
848 sdi.ImmediateData = 0;
849 }
850 }
851 } else {
852 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_DATA_IMM), sdi) {
853 sdi.Address = addr;
854 if (GEN_GEN >= 8 || GEN_IS_HASWELL) {
855 /* Pre-SKL, the dword containing the clear values also contains
856 * other fields, so we need to initialize those fields to match the
857 * values that would be in a color attachment.
858 */
859 sdi.ImmediateData = ISL_CHANNEL_SELECT_RED << 25 |
860 ISL_CHANNEL_SELECT_GREEN << 22 |
861 ISL_CHANNEL_SELECT_BLUE << 19 |
862 ISL_CHANNEL_SELECT_ALPHA << 16;
863 } else if (GEN_GEN == 7) {
864 /* On IVB, the dword containing the clear values also contains
865 * other fields that must be zero or can be zero.
866 */
867 sdi.ImmediateData = 0;
868 }
869 }
870 }
871 }
872
873 /* Copy the fast-clear value dword(s) between a surface state object and an
874 * image's fast clear state buffer.
875 */
876 static void
877 genX(copy_fast_clear_dwords)(struct anv_cmd_buffer *cmd_buffer,
878 struct anv_state surface_state,
879 const struct anv_image *image,
880 VkImageAspectFlagBits aspect,
881 bool copy_from_surface_state)
882 {
883 assert(cmd_buffer && image);
884 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
885
886 struct anv_address ss_clear_addr = {
887 .bo = cmd_buffer->device->surface_state_pool.block_pool.bo,
888 .offset = surface_state.offset +
889 cmd_buffer->device->isl_dev.ss.clear_value_offset,
890 };
891 const struct anv_address entry_addr =
892 anv_image_get_clear_color_addr(cmd_buffer->device, image, aspect);
893 unsigned copy_size = cmd_buffer->device->isl_dev.ss.clear_value_size;
894
895 #if GEN_GEN == 7
896 /* On gen7, the combination of commands used here(MI_LOAD_REGISTER_MEM
897 * and MI_STORE_REGISTER_MEM) can cause GPU hangs if any rendering is
898 * in-flight when they are issued even if the memory touched is not
899 * currently active for rendering. The weird bit is that it is not the
900 * MI_LOAD/STORE_REGISTER_MEM commands which hang but rather the in-flight
901 * rendering hangs such that the next stalling command after the
902 * MI_LOAD/STORE_REGISTER_MEM commands will catch the hang.
903 *
904 * It is unclear exactly why this hang occurs. Both MI commands come with
905 * warnings about the 3D pipeline but that doesn't seem to fully explain
906 * it. My (Jason's) best theory is that it has something to do with the
907 * fact that we're using a GPU state register as our temporary and that
908 * something with reading/writing it is causing problems.
909 *
910 * In order to work around this issue, we emit a PIPE_CONTROL with the
911 * command streamer stall bit set.
912 */
913 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_CS_STALL_BIT;
914 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
915 #endif
916
917 struct gen_mi_builder b;
918 gen_mi_builder_init(&b, &cmd_buffer->batch);
919
920 if (copy_from_surface_state) {
921 gen_mi_memcpy(&b, entry_addr, ss_clear_addr, copy_size);
922 } else {
923 gen_mi_memcpy(&b, ss_clear_addr, entry_addr, copy_size);
924
925 /* Updating a surface state object may require that the state cache be
926 * invalidated. From the SKL PRM, Shared Functions -> State -> State
927 * Caching:
928 *
929 * Whenever the RENDER_SURFACE_STATE object in memory pointed to by
930 * the Binding Table Pointer (BTP) and Binding Table Index (BTI) is
931 * modified [...], the L1 state cache must be invalidated to ensure
932 * the new surface or sampler state is fetched from system memory.
933 *
934 * In testing, SKL doesn't actually seem to need this, but HSW does.
935 */
936 cmd_buffer->state.pending_pipe_bits |=
937 ANV_PIPE_STATE_CACHE_INVALIDATE_BIT;
938 }
939 }
940
941 /**
942 * @brief Transitions a color buffer from one layout to another.
943 *
944 * See section 6.1.1. Image Layout Transitions of the Vulkan 1.0.50 spec for
945 * more information.
946 *
947 * @param level_count VK_REMAINING_MIP_LEVELS isn't supported.
948 * @param layer_count VK_REMAINING_ARRAY_LAYERS isn't supported. For 3D images,
949 * this represents the maximum layers to transition at each
950 * specified miplevel.
951 */
952 static void
953 transition_color_buffer(struct anv_cmd_buffer *cmd_buffer,
954 const struct anv_image *image,
955 VkImageAspectFlagBits aspect,
956 const uint32_t base_level, uint32_t level_count,
957 uint32_t base_layer, uint32_t layer_count,
958 VkImageLayout initial_layout,
959 VkImageLayout final_layout)
960 {
961 const struct gen_device_info *devinfo = &cmd_buffer->device->info;
962 /* Validate the inputs. */
963 assert(cmd_buffer);
964 assert(image && image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
965 /* These values aren't supported for simplicity's sake. */
966 assert(level_count != VK_REMAINING_MIP_LEVELS &&
967 layer_count != VK_REMAINING_ARRAY_LAYERS);
968 /* Ensure the subresource range is valid. */
969 UNUSED uint64_t last_level_num = base_level + level_count;
970 const uint32_t max_depth = anv_minify(image->extent.depth, base_level);
971 UNUSED const uint32_t image_layers = MAX2(image->array_size, max_depth);
972 assert((uint64_t)base_layer + layer_count <= image_layers);
973 assert(last_level_num <= image->levels);
974 /* The spec disallows these final layouts. */
975 assert(final_layout != VK_IMAGE_LAYOUT_UNDEFINED &&
976 final_layout != VK_IMAGE_LAYOUT_PREINITIALIZED);
977
978 /* No work is necessary if the layout stays the same or if this subresource
979 * range lacks auxiliary data.
980 */
981 if (initial_layout == final_layout)
982 return;
983
984 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
985
986 if (image->planes[plane].shadow_surface.isl.size_B > 0 &&
987 final_layout == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) {
988 /* This surface is a linear compressed image with a tiled shadow surface
989 * for texturing. The client is about to use it in READ_ONLY_OPTIMAL so
990 * we need to ensure the shadow copy is up-to-date.
991 */
992 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT);
993 assert(image->planes[plane].surface.isl.tiling == ISL_TILING_LINEAR);
994 assert(image->planes[plane].shadow_surface.isl.tiling != ISL_TILING_LINEAR);
995 assert(isl_format_is_compressed(image->planes[plane].surface.isl.format));
996 assert(plane == 0);
997 anv_image_copy_to_shadow(cmd_buffer, image,
998 VK_IMAGE_ASPECT_COLOR_BIT,
999 base_level, level_count,
1000 base_layer, layer_count);
1001 }
1002
1003 if (base_layer >= anv_image_aux_layers(image, aspect, base_level))
1004 return;
1005
1006 assert(image->tiling == VK_IMAGE_TILING_OPTIMAL);
1007
1008 if (initial_layout == VK_IMAGE_LAYOUT_UNDEFINED ||
1009 initial_layout == VK_IMAGE_LAYOUT_PREINITIALIZED) {
1010 /* A subresource in the undefined layout may have been aliased and
1011 * populated with any arrangement of bits. Therefore, we must initialize
1012 * the related aux buffer and clear buffer entry with desirable values.
1013 * An initial layout of PREINITIALIZED is the same as UNDEFINED for
1014 * images with VK_IMAGE_TILING_OPTIMAL.
1015 *
1016 * Initialize the relevant clear buffer entries.
1017 */
1018 if (base_level == 0 && base_layer == 0)
1019 init_fast_clear_color(cmd_buffer, image, aspect);
1020
1021 /* Initialize the aux buffers to enable correct rendering. In order to
1022 * ensure that things such as storage images work correctly, aux buffers
1023 * need to be initialized to valid data.
1024 *
1025 * Having an aux buffer with invalid data is a problem for two reasons:
1026 *
1027 * 1) Having an invalid value in the buffer can confuse the hardware.
1028 * For instance, with CCS_E on SKL, a two-bit CCS value of 2 is
1029 * invalid and leads to the hardware doing strange things. It
1030 * doesn't hang as far as we can tell but rendering corruption can
1031 * occur.
1032 *
1033 * 2) If this transition is into the GENERAL layout and we then use the
1034 * image as a storage image, then we must have the aux buffer in the
1035 * pass-through state so that, if we then go to texture from the
1036 * image, we get the results of our storage image writes and not the
1037 * fast clear color or other random data.
1038 *
1039 * For CCS both of the problems above are real demonstrable issues. In
1040 * that case, the only thing we can do is to perform an ambiguate to
1041 * transition the aux surface into the pass-through state.
1042 *
1043 * For MCS, (2) is never an issue because we don't support multisampled
1044 * storage images. In theory, issue (1) is a problem with MCS but we've
1045 * never seen it in the wild. For 4x and 16x, all bit patters could, in
1046 * theory, be interpreted as something but we don't know that all bit
1047 * patterns are actually valid. For 2x and 8x, you could easily end up
1048 * with the MCS referring to an invalid plane because not all bits of
1049 * the MCS value are actually used. Even though we've never seen issues
1050 * in the wild, it's best to play it safe and initialize the MCS. We
1051 * can use a fast-clear for MCS because we only ever touch from render
1052 * and texture (no image load store).
1053 */
1054 if (image->samples == 1) {
1055 for (uint32_t l = 0; l < level_count; l++) {
1056 const uint32_t level = base_level + l;
1057
1058 uint32_t aux_layers = anv_image_aux_layers(image, aspect, level);
1059 if (base_layer >= aux_layers)
1060 break; /* We will only get fewer layers as level increases */
1061 uint32_t level_layer_count =
1062 MIN2(layer_count, aux_layers - base_layer);
1063
1064 anv_image_ccs_op(cmd_buffer, image,
1065 image->planes[plane].surface.isl.format,
1066 aspect, level, base_layer, level_layer_count,
1067 ISL_AUX_OP_AMBIGUATE, NULL, false);
1068
1069 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_CCS_E) {
1070 set_image_compressed_bit(cmd_buffer, image, aspect,
1071 level, base_layer, level_layer_count,
1072 false);
1073 }
1074 }
1075 } else {
1076 if (image->samples == 4 || image->samples == 16) {
1077 anv_perf_warn(cmd_buffer->device->instance, image,
1078 "Doing a potentially unnecessary fast-clear to "
1079 "define an MCS buffer.");
1080 }
1081
1082 assert(base_level == 0 && level_count == 1);
1083 anv_image_mcs_op(cmd_buffer, image,
1084 image->planes[plane].surface.isl.format,
1085 aspect, base_layer, layer_count,
1086 ISL_AUX_OP_FAST_CLEAR, NULL, false);
1087 }
1088 return;
1089 }
1090
1091 const enum isl_aux_usage initial_aux_usage =
1092 anv_layout_to_aux_usage(devinfo, image, aspect, initial_layout);
1093 const enum isl_aux_usage final_aux_usage =
1094 anv_layout_to_aux_usage(devinfo, image, aspect, final_layout);
1095
1096 /* The current code assumes that there is no mixing of CCS_E and CCS_D.
1097 * We can handle transitions between CCS_D/E to and from NONE. What we
1098 * don't yet handle is switching between CCS_E and CCS_D within a given
1099 * image. Doing so in a performant way requires more detailed aux state
1100 * tracking such as what is done in i965. For now, just assume that we
1101 * only have one type of compression.
1102 */
1103 assert(initial_aux_usage == ISL_AUX_USAGE_NONE ||
1104 final_aux_usage == ISL_AUX_USAGE_NONE ||
1105 initial_aux_usage == final_aux_usage);
1106
1107 /* If initial aux usage is NONE, there is nothing to resolve */
1108 if (initial_aux_usage == ISL_AUX_USAGE_NONE)
1109 return;
1110
1111 enum isl_aux_op resolve_op = ISL_AUX_OP_NONE;
1112
1113 /* If the initial layout supports more fast clear than the final layout
1114 * then we need at least a partial resolve.
1115 */
1116 const enum anv_fast_clear_type initial_fast_clear =
1117 anv_layout_to_fast_clear_type(devinfo, image, aspect, initial_layout);
1118 const enum anv_fast_clear_type final_fast_clear =
1119 anv_layout_to_fast_clear_type(devinfo, image, aspect, final_layout);
1120 if (final_fast_clear < initial_fast_clear)
1121 resolve_op = ISL_AUX_OP_PARTIAL_RESOLVE;
1122
1123 if (initial_aux_usage == ISL_AUX_USAGE_CCS_E &&
1124 final_aux_usage != ISL_AUX_USAGE_CCS_E)
1125 resolve_op = ISL_AUX_OP_FULL_RESOLVE;
1126
1127 if (resolve_op == ISL_AUX_OP_NONE)
1128 return;
1129
1130 /* Perform a resolve to synchronize data between the main and aux buffer.
1131 * Before we begin, we must satisfy the cache flushing requirement specified
1132 * in the Sky Lake PRM Vol. 7, "MCS Buffer for Render Target(s)":
1133 *
1134 * Any transition from any value in {Clear, Render, Resolve} to a
1135 * different value in {Clear, Render, Resolve} requires end of pipe
1136 * synchronization.
1137 *
1138 * We perform a flush of the write cache before and after the clear and
1139 * resolve operations to meet this requirement.
1140 *
1141 * Unlike other drawing, fast clear operations are not properly
1142 * synchronized. The first PIPE_CONTROL here likely ensures that the
1143 * contents of the previous render or clear hit the render target before we
1144 * resolve and the second likely ensures that the resolve is complete before
1145 * we do any more rendering or clearing.
1146 */
1147 cmd_buffer->state.pending_pipe_bits |=
1148 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1149
1150 for (uint32_t l = 0; l < level_count; l++) {
1151 uint32_t level = base_level + l;
1152
1153 uint32_t aux_layers = anv_image_aux_layers(image, aspect, level);
1154 if (base_layer >= aux_layers)
1155 break; /* We will only get fewer layers as level increases */
1156 uint32_t level_layer_count =
1157 MIN2(layer_count, aux_layers - base_layer);
1158
1159 for (uint32_t a = 0; a < level_layer_count; a++) {
1160 uint32_t array_layer = base_layer + a;
1161 if (image->samples == 1) {
1162 anv_cmd_predicated_ccs_resolve(cmd_buffer, image,
1163 image->planes[plane].surface.isl.format,
1164 aspect, level, array_layer, resolve_op,
1165 final_fast_clear);
1166 } else {
1167 /* We only support fast-clear on the first layer so partial
1168 * resolves should not be used on other layers as they will use
1169 * the clear color stored in memory that is only valid for layer0.
1170 */
1171 if (resolve_op == ISL_AUX_OP_PARTIAL_RESOLVE &&
1172 array_layer != 0)
1173 continue;
1174
1175 anv_cmd_predicated_mcs_resolve(cmd_buffer, image,
1176 image->planes[plane].surface.isl.format,
1177 aspect, array_layer, resolve_op,
1178 final_fast_clear);
1179 }
1180 }
1181 }
1182
1183 cmd_buffer->state.pending_pipe_bits |=
1184 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT;
1185 }
1186
1187 /**
1188 * Setup anv_cmd_state::attachments for vkCmdBeginRenderPass.
1189 */
1190 static VkResult
1191 genX(cmd_buffer_setup_attachments)(struct anv_cmd_buffer *cmd_buffer,
1192 struct anv_render_pass *pass,
1193 const VkRenderPassBeginInfo *begin)
1194 {
1195 const struct isl_device *isl_dev = &cmd_buffer->device->isl_dev;
1196 struct anv_cmd_state *state = &cmd_buffer->state;
1197 struct anv_framebuffer *framebuffer = cmd_buffer->state.framebuffer;
1198
1199 vk_free(&cmd_buffer->pool->alloc, state->attachments);
1200
1201 if (pass->attachment_count > 0) {
1202 state->attachments = vk_alloc(&cmd_buffer->pool->alloc,
1203 pass->attachment_count *
1204 sizeof(state->attachments[0]),
1205 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1206 if (state->attachments == NULL) {
1207 /* Propagate VK_ERROR_OUT_OF_HOST_MEMORY to vkEndCommandBuffer */
1208 return anv_batch_set_error(&cmd_buffer->batch,
1209 VK_ERROR_OUT_OF_HOST_MEMORY);
1210 }
1211 } else {
1212 state->attachments = NULL;
1213 }
1214
1215 /* Reserve one for the NULL state. */
1216 unsigned num_states = 1;
1217 for (uint32_t i = 0; i < pass->attachment_count; ++i) {
1218 if (vk_format_is_color(pass->attachments[i].format))
1219 num_states++;
1220
1221 if (need_input_attachment_state(&pass->attachments[i]))
1222 num_states++;
1223 }
1224
1225 const uint32_t ss_stride = align_u32(isl_dev->ss.size, isl_dev->ss.align);
1226 state->render_pass_states =
1227 anv_state_stream_alloc(&cmd_buffer->surface_state_stream,
1228 num_states * ss_stride, isl_dev->ss.align);
1229
1230 struct anv_state next_state = state->render_pass_states;
1231 next_state.alloc_size = isl_dev->ss.size;
1232
1233 state->null_surface_state = next_state;
1234 next_state.offset += ss_stride;
1235 next_state.map += ss_stride;
1236
1237 const VkRenderPassAttachmentBeginInfoKHR *begin_attachment =
1238 vk_find_struct_const(begin, RENDER_PASS_ATTACHMENT_BEGIN_INFO_KHR);
1239
1240 if (begin && !begin_attachment)
1241 assert(pass->attachment_count == framebuffer->attachment_count);
1242
1243 for (uint32_t i = 0; i < pass->attachment_count; ++i) {
1244 if (vk_format_is_color(pass->attachments[i].format)) {
1245 state->attachments[i].color.state = next_state;
1246 next_state.offset += ss_stride;
1247 next_state.map += ss_stride;
1248 }
1249
1250 if (need_input_attachment_state(&pass->attachments[i])) {
1251 state->attachments[i].input.state = next_state;
1252 next_state.offset += ss_stride;
1253 next_state.map += ss_stride;
1254 }
1255
1256 if (begin_attachment && begin_attachment->attachmentCount != 0) {
1257 assert(begin_attachment->attachmentCount == pass->attachment_count);
1258 ANV_FROM_HANDLE(anv_image_view, iview, begin_attachment->pAttachments[i]);
1259 cmd_buffer->state.attachments[i].image_view = iview;
1260 } else if (framebuffer && i < framebuffer->attachment_count) {
1261 cmd_buffer->state.attachments[i].image_view = framebuffer->attachments[i];
1262 }
1263 }
1264 assert(next_state.offset == state->render_pass_states.offset +
1265 state->render_pass_states.alloc_size);
1266
1267 if (begin) {
1268 isl_null_fill_state(isl_dev, state->null_surface_state.map,
1269 isl_extent3d(framebuffer->width,
1270 framebuffer->height,
1271 framebuffer->layers));
1272
1273 for (uint32_t i = 0; i < pass->attachment_count; ++i) {
1274 struct anv_render_pass_attachment *att = &pass->attachments[i];
1275 VkImageAspectFlags att_aspects = vk_format_aspects(att->format);
1276 VkImageAspectFlags clear_aspects = 0;
1277 VkImageAspectFlags load_aspects = 0;
1278
1279 if (att_aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1280 /* color attachment */
1281 if (att->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
1282 clear_aspects |= VK_IMAGE_ASPECT_COLOR_BIT;
1283 } else if (att->load_op == VK_ATTACHMENT_LOAD_OP_LOAD) {
1284 load_aspects |= VK_IMAGE_ASPECT_COLOR_BIT;
1285 }
1286 } else {
1287 /* depthstencil attachment */
1288 if (att_aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
1289 if (att->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
1290 clear_aspects |= VK_IMAGE_ASPECT_DEPTH_BIT;
1291 } else if (att->load_op == VK_ATTACHMENT_LOAD_OP_LOAD) {
1292 load_aspects |= VK_IMAGE_ASPECT_DEPTH_BIT;
1293 }
1294 }
1295 if (att_aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1296 if (att->stencil_load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
1297 clear_aspects |= VK_IMAGE_ASPECT_STENCIL_BIT;
1298 } else if (att->stencil_load_op == VK_ATTACHMENT_LOAD_OP_LOAD) {
1299 load_aspects |= VK_IMAGE_ASPECT_STENCIL_BIT;
1300 }
1301 }
1302 }
1303
1304 state->attachments[i].current_layout = att->initial_layout;
1305 state->attachments[i].pending_clear_aspects = clear_aspects;
1306 state->attachments[i].pending_load_aspects = load_aspects;
1307 if (clear_aspects)
1308 state->attachments[i].clear_value = begin->pClearValues[i];
1309
1310 struct anv_image_view *iview = cmd_buffer->state.attachments[i].image_view;
1311 anv_assert(iview->vk_format == att->format);
1312
1313 const uint32_t num_layers = iview->planes[0].isl.array_len;
1314 state->attachments[i].pending_clear_views = (1 << num_layers) - 1;
1315
1316 union isl_color_value clear_color = { .u32 = { 0, } };
1317 if (att_aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1318 anv_assert(iview->n_planes == 1);
1319 assert(att_aspects == VK_IMAGE_ASPECT_COLOR_BIT);
1320 color_attachment_compute_aux_usage(cmd_buffer->device,
1321 state, i, begin->renderArea,
1322 &clear_color);
1323
1324 anv_image_fill_surface_state(cmd_buffer->device,
1325 iview->image,
1326 VK_IMAGE_ASPECT_COLOR_BIT,
1327 &iview->planes[0].isl,
1328 ISL_SURF_USAGE_RENDER_TARGET_BIT,
1329 state->attachments[i].aux_usage,
1330 &clear_color,
1331 0,
1332 &state->attachments[i].color,
1333 NULL);
1334
1335 add_surface_state_relocs(cmd_buffer, state->attachments[i].color);
1336 } else {
1337 depth_stencil_attachment_compute_aux_usage(cmd_buffer->device,
1338 state, i,
1339 begin->renderArea);
1340 }
1341
1342 if (need_input_attachment_state(&pass->attachments[i])) {
1343 anv_image_fill_surface_state(cmd_buffer->device,
1344 iview->image,
1345 VK_IMAGE_ASPECT_COLOR_BIT,
1346 &iview->planes[0].isl,
1347 ISL_SURF_USAGE_TEXTURE_BIT,
1348 state->attachments[i].input_aux_usage,
1349 &clear_color,
1350 0,
1351 &state->attachments[i].input,
1352 NULL);
1353
1354 add_surface_state_relocs(cmd_buffer, state->attachments[i].input);
1355 }
1356 }
1357 }
1358
1359 return VK_SUCCESS;
1360 }
1361
1362 VkResult
1363 genX(BeginCommandBuffer)(
1364 VkCommandBuffer commandBuffer,
1365 const VkCommandBufferBeginInfo* pBeginInfo)
1366 {
1367 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1368
1369 /* If this is the first vkBeginCommandBuffer, we must *initialize* the
1370 * command buffer's state. Otherwise, we must *reset* its state. In both
1371 * cases we reset it.
1372 *
1373 * From the Vulkan 1.0 spec:
1374 *
1375 * If a command buffer is in the executable state and the command buffer
1376 * was allocated from a command pool with the
1377 * VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT flag set, then
1378 * vkBeginCommandBuffer implicitly resets the command buffer, behaving
1379 * as if vkResetCommandBuffer had been called with
1380 * VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT not set. It then puts
1381 * the command buffer in the recording state.
1382 */
1383 anv_cmd_buffer_reset(cmd_buffer);
1384
1385 cmd_buffer->usage_flags = pBeginInfo->flags;
1386
1387 assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY ||
1388 !(cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT));
1389
1390 genX(cmd_buffer_emit_state_base_address)(cmd_buffer);
1391
1392 /* We sometimes store vertex data in the dynamic state buffer for blorp
1393 * operations and our dynamic state stream may re-use data from previous
1394 * command buffers. In order to prevent stale cache data, we flush the VF
1395 * cache. We could do this on every blorp call but that's not really
1396 * needed as all of the data will get written by the CPU prior to the GPU
1397 * executing anything. The chances are fairly high that they will use
1398 * blorp at least once per primary command buffer so it shouldn't be
1399 * wasted.
1400 */
1401 if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY)
1402 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
1403
1404 /* We send an "Indirect State Pointers Disable" packet at
1405 * EndCommandBuffer, so all push contant packets are ignored during a
1406 * context restore. Documentation says after that command, we need to
1407 * emit push constants again before any rendering operation. So we
1408 * flag them dirty here to make sure they get emitted.
1409 */
1410 cmd_buffer->state.push_constants_dirty |= VK_SHADER_STAGE_ALL_GRAPHICS;
1411
1412 VkResult result = VK_SUCCESS;
1413 if (cmd_buffer->usage_flags &
1414 VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) {
1415 assert(pBeginInfo->pInheritanceInfo);
1416 cmd_buffer->state.pass =
1417 anv_render_pass_from_handle(pBeginInfo->pInheritanceInfo->renderPass);
1418 cmd_buffer->state.subpass =
1419 &cmd_buffer->state.pass->subpasses[pBeginInfo->pInheritanceInfo->subpass];
1420
1421 /* This is optional in the inheritance info. */
1422 cmd_buffer->state.framebuffer =
1423 anv_framebuffer_from_handle(pBeginInfo->pInheritanceInfo->framebuffer);
1424
1425 result = genX(cmd_buffer_setup_attachments)(cmd_buffer,
1426 cmd_buffer->state.pass, NULL);
1427
1428 /* Record that HiZ is enabled if we can. */
1429 if (cmd_buffer->state.framebuffer) {
1430 const struct anv_image_view * const iview =
1431 anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
1432
1433 if (iview) {
1434 VkImageLayout layout =
1435 cmd_buffer->state.subpass->depth_stencil_attachment->layout;
1436
1437 enum isl_aux_usage aux_usage =
1438 anv_layout_to_aux_usage(&cmd_buffer->device->info, iview->image,
1439 VK_IMAGE_ASPECT_DEPTH_BIT, layout);
1440
1441 cmd_buffer->state.hiz_enabled = aux_usage == ISL_AUX_USAGE_HIZ;
1442 }
1443 }
1444
1445 cmd_buffer->state.gfx.dirty |= ANV_CMD_DIRTY_RENDER_TARGETS;
1446 }
1447
1448 #if GEN_GEN >= 8 || GEN_IS_HASWELL
1449 if (cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY) {
1450 const VkCommandBufferInheritanceConditionalRenderingInfoEXT *conditional_rendering_info =
1451 vk_find_struct_const(pBeginInfo->pInheritanceInfo->pNext, COMMAND_BUFFER_INHERITANCE_CONDITIONAL_RENDERING_INFO_EXT);
1452
1453 /* If secondary buffer supports conditional rendering
1454 * we should emit commands as if conditional rendering is enabled.
1455 */
1456 cmd_buffer->state.conditional_render_enabled =
1457 conditional_rendering_info && conditional_rendering_info->conditionalRenderingEnable;
1458 }
1459 #endif
1460
1461 return result;
1462 }
1463
1464 /* From the PRM, Volume 2a:
1465 *
1466 * "Indirect State Pointers Disable
1467 *
1468 * At the completion of the post-sync operation associated with this pipe
1469 * control packet, the indirect state pointers in the hardware are
1470 * considered invalid; the indirect pointers are not saved in the context.
1471 * If any new indirect state commands are executed in the command stream
1472 * while the pipe control is pending, the new indirect state commands are
1473 * preserved.
1474 *
1475 * [DevIVB+]: Using Invalidate State Pointer (ISP) only inhibits context
1476 * restoring of Push Constant (3DSTATE_CONSTANT_*) commands. Push Constant
1477 * commands are only considered as Indirect State Pointers. Once ISP is
1478 * issued in a context, SW must initialize by programming push constant
1479 * commands for all the shaders (at least to zero length) before attempting
1480 * any rendering operation for the same context."
1481 *
1482 * 3DSTATE_CONSTANT_* packets are restored during a context restore,
1483 * even though they point to a BO that has been already unreferenced at
1484 * the end of the previous batch buffer. This has been fine so far since
1485 * we are protected by these scratch page (every address not covered by
1486 * a BO should be pointing to the scratch page). But on CNL, it is
1487 * causing a GPU hang during context restore at the 3DSTATE_CONSTANT_*
1488 * instruction.
1489 *
1490 * The flag "Indirect State Pointers Disable" in PIPE_CONTROL tells the
1491 * hardware to ignore previous 3DSTATE_CONSTANT_* packets during a
1492 * context restore, so the mentioned hang doesn't happen. However,
1493 * software must program push constant commands for all stages prior to
1494 * rendering anything. So we flag them dirty in BeginCommandBuffer.
1495 *
1496 * Finally, we also make sure to stall at pixel scoreboard to make sure the
1497 * constants have been loaded into the EUs prior to disable the push constants
1498 * so that it doesn't hang a previous 3DPRIMITIVE.
1499 */
1500 static void
1501 emit_isp_disable(struct anv_cmd_buffer *cmd_buffer)
1502 {
1503 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
1504 pc.StallAtPixelScoreboard = true;
1505 pc.CommandStreamerStallEnable = true;
1506 }
1507 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
1508 pc.IndirectStatePointersDisable = true;
1509 pc.CommandStreamerStallEnable = true;
1510 }
1511 }
1512
1513 VkResult
1514 genX(EndCommandBuffer)(
1515 VkCommandBuffer commandBuffer)
1516 {
1517 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1518
1519 if (anv_batch_has_error(&cmd_buffer->batch))
1520 return cmd_buffer->batch.status;
1521
1522 /* We want every command buffer to start with the PMA fix in a known state,
1523 * so we disable it at the end of the command buffer.
1524 */
1525 genX(cmd_buffer_enable_pma_fix)(cmd_buffer, false);
1526
1527 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
1528
1529 emit_isp_disable(cmd_buffer);
1530
1531 anv_cmd_buffer_end_batch_buffer(cmd_buffer);
1532
1533 return VK_SUCCESS;
1534 }
1535
1536 void
1537 genX(CmdExecuteCommands)(
1538 VkCommandBuffer commandBuffer,
1539 uint32_t commandBufferCount,
1540 const VkCommandBuffer* pCmdBuffers)
1541 {
1542 ANV_FROM_HANDLE(anv_cmd_buffer, primary, commandBuffer);
1543
1544 assert(primary->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
1545
1546 if (anv_batch_has_error(&primary->batch))
1547 return;
1548
1549 /* The secondary command buffers will assume that the PMA fix is disabled
1550 * when they begin executing. Make sure this is true.
1551 */
1552 genX(cmd_buffer_enable_pma_fix)(primary, false);
1553
1554 /* The secondary command buffer doesn't know which textures etc. have been
1555 * flushed prior to their execution. Apply those flushes now.
1556 */
1557 genX(cmd_buffer_apply_pipe_flushes)(primary);
1558
1559 for (uint32_t i = 0; i < commandBufferCount; i++) {
1560 ANV_FROM_HANDLE(anv_cmd_buffer, secondary, pCmdBuffers[i]);
1561
1562 assert(secondary->level == VK_COMMAND_BUFFER_LEVEL_SECONDARY);
1563 assert(!anv_batch_has_error(&secondary->batch));
1564
1565 #if GEN_GEN >= 8 || GEN_IS_HASWELL
1566 if (secondary->state.conditional_render_enabled) {
1567 if (!primary->state.conditional_render_enabled) {
1568 /* Secondary buffer is constructed as if it will be executed
1569 * with conditional rendering, we should satisfy this dependency
1570 * regardless of conditional rendering being enabled in primary.
1571 */
1572 struct gen_mi_builder b;
1573 gen_mi_builder_init(&b, &primary->batch);
1574 gen_mi_store(&b, gen_mi_reg64(ANV_PREDICATE_RESULT_REG),
1575 gen_mi_imm(UINT64_MAX));
1576 }
1577 }
1578 #endif
1579
1580 if (secondary->usage_flags &
1581 VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT) {
1582 /* If we're continuing a render pass from the primary, we need to
1583 * copy the surface states for the current subpass into the storage
1584 * we allocated for them in BeginCommandBuffer.
1585 */
1586 struct anv_bo *ss_bo =
1587 primary->device->surface_state_pool.block_pool.bo;
1588 struct anv_state src_state = primary->state.render_pass_states;
1589 struct anv_state dst_state = secondary->state.render_pass_states;
1590 assert(src_state.alloc_size == dst_state.alloc_size);
1591
1592 genX(cmd_buffer_so_memcpy)(primary,
1593 (struct anv_address) {
1594 .bo = ss_bo,
1595 .offset = dst_state.offset,
1596 },
1597 (struct anv_address) {
1598 .bo = ss_bo,
1599 .offset = src_state.offset,
1600 },
1601 src_state.alloc_size);
1602 }
1603
1604 anv_cmd_buffer_add_secondary(primary, secondary);
1605 }
1606
1607 /* The secondary may have selected a different pipeline (3D or compute) and
1608 * may have changed the current L3$ configuration. Reset our tracking
1609 * variables to invalid values to ensure that we re-emit these in the case
1610 * where we do any draws or compute dispatches from the primary after the
1611 * secondary has returned.
1612 */
1613 primary->state.current_pipeline = UINT32_MAX;
1614 primary->state.current_l3_config = NULL;
1615
1616 /* Each of the secondary command buffers will use its own state base
1617 * address. We need to re-emit state base address for the primary after
1618 * all of the secondaries are done.
1619 *
1620 * TODO: Maybe we want to make this a dirty bit to avoid extra state base
1621 * address calls?
1622 */
1623 genX(cmd_buffer_emit_state_base_address)(primary);
1624 }
1625
1626 #define IVB_L3SQCREG1_SQGHPCI_DEFAULT 0x00730000
1627 #define VLV_L3SQCREG1_SQGHPCI_DEFAULT 0x00d30000
1628 #define HSW_L3SQCREG1_SQGHPCI_DEFAULT 0x00610000
1629
1630 /**
1631 * Program the hardware to use the specified L3 configuration.
1632 */
1633 void
1634 genX(cmd_buffer_config_l3)(struct anv_cmd_buffer *cmd_buffer,
1635 const struct gen_l3_config *cfg)
1636 {
1637 assert(cfg);
1638 if (cfg == cmd_buffer->state.current_l3_config)
1639 return;
1640
1641 if (unlikely(INTEL_DEBUG & DEBUG_L3)) {
1642 intel_logd("L3 config transition: ");
1643 gen_dump_l3_config(cfg, stderr);
1644 }
1645
1646 const bool has_slm = cfg->n[GEN_L3P_SLM];
1647
1648 /* According to the hardware docs, the L3 partitioning can only be changed
1649 * while the pipeline is completely drained and the caches are flushed,
1650 * which involves a first PIPE_CONTROL flush which stalls the pipeline...
1651 */
1652 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
1653 pc.DCFlushEnable = true;
1654 pc.PostSyncOperation = NoWrite;
1655 pc.CommandStreamerStallEnable = true;
1656 }
1657
1658 /* ...followed by a second pipelined PIPE_CONTROL that initiates
1659 * invalidation of the relevant caches. Note that because RO invalidation
1660 * happens at the top of the pipeline (i.e. right away as the PIPE_CONTROL
1661 * command is processed by the CS) we cannot combine it with the previous
1662 * stalling flush as the hardware documentation suggests, because that
1663 * would cause the CS to stall on previous rendering *after* RO
1664 * invalidation and wouldn't prevent the RO caches from being polluted by
1665 * concurrent rendering before the stall completes. This intentionally
1666 * doesn't implement the SKL+ hardware workaround suggesting to enable CS
1667 * stall on PIPE_CONTROLs with the texture cache invalidation bit set for
1668 * GPGPU workloads because the previous and subsequent PIPE_CONTROLs
1669 * already guarantee that there is no concurrent GPGPU kernel execution
1670 * (see SKL HSD 2132585).
1671 */
1672 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
1673 pc.TextureCacheInvalidationEnable = true;
1674 pc.ConstantCacheInvalidationEnable = true;
1675 pc.InstructionCacheInvalidateEnable = true;
1676 pc.StateCacheInvalidationEnable = true;
1677 pc.PostSyncOperation = NoWrite;
1678 }
1679
1680 /* Now send a third stalling flush to make sure that invalidation is
1681 * complete when the L3 configuration registers are modified.
1682 */
1683 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
1684 pc.DCFlushEnable = true;
1685 pc.PostSyncOperation = NoWrite;
1686 pc.CommandStreamerStallEnable = true;
1687 }
1688
1689 #if GEN_GEN >= 8
1690
1691 assert(!cfg->n[GEN_L3P_IS] && !cfg->n[GEN_L3P_C] && !cfg->n[GEN_L3P_T]);
1692
1693 uint32_t l3cr;
1694 anv_pack_struct(&l3cr, GENX(L3CNTLREG),
1695 .SLMEnable = has_slm,
1696 #if GEN_GEN == 11
1697 /* WA_1406697149: Bit 9 "Error Detection Behavior Control" must be set
1698 * in L3CNTLREG register. The default setting of the bit is not the
1699 * desirable behavior.
1700 */
1701 .ErrorDetectionBehaviorControl = true,
1702 .UseFullWays = true,
1703 #endif
1704 .URBAllocation = cfg->n[GEN_L3P_URB],
1705 .ROAllocation = cfg->n[GEN_L3P_RO],
1706 .DCAllocation = cfg->n[GEN_L3P_DC],
1707 .AllAllocation = cfg->n[GEN_L3P_ALL]);
1708
1709 /* Set up the L3 partitioning. */
1710 emit_lri(&cmd_buffer->batch, GENX(L3CNTLREG_num), l3cr);
1711
1712 #else
1713
1714 const bool has_dc = cfg->n[GEN_L3P_DC] || cfg->n[GEN_L3P_ALL];
1715 const bool has_is = cfg->n[GEN_L3P_IS] || cfg->n[GEN_L3P_RO] ||
1716 cfg->n[GEN_L3P_ALL];
1717 const bool has_c = cfg->n[GEN_L3P_C] || cfg->n[GEN_L3P_RO] ||
1718 cfg->n[GEN_L3P_ALL];
1719 const bool has_t = cfg->n[GEN_L3P_T] || cfg->n[GEN_L3P_RO] ||
1720 cfg->n[GEN_L3P_ALL];
1721
1722 assert(!cfg->n[GEN_L3P_ALL]);
1723
1724 /* When enabled SLM only uses a portion of the L3 on half of the banks,
1725 * the matching space on the remaining banks has to be allocated to a
1726 * client (URB for all validated configurations) set to the
1727 * lower-bandwidth 2-bank address hashing mode.
1728 */
1729 const struct gen_device_info *devinfo = &cmd_buffer->device->info;
1730 const bool urb_low_bw = has_slm && !devinfo->is_baytrail;
1731 assert(!urb_low_bw || cfg->n[GEN_L3P_URB] == cfg->n[GEN_L3P_SLM]);
1732
1733 /* Minimum number of ways that can be allocated to the URB. */
1734 const unsigned n0_urb = devinfo->is_baytrail ? 32 : 0;
1735 assert(cfg->n[GEN_L3P_URB] >= n0_urb);
1736
1737 uint32_t l3sqcr1, l3cr2, l3cr3;
1738 anv_pack_struct(&l3sqcr1, GENX(L3SQCREG1),
1739 .ConvertDC_UC = !has_dc,
1740 .ConvertIS_UC = !has_is,
1741 .ConvertC_UC = !has_c,
1742 .ConvertT_UC = !has_t);
1743 l3sqcr1 |=
1744 GEN_IS_HASWELL ? HSW_L3SQCREG1_SQGHPCI_DEFAULT :
1745 devinfo->is_baytrail ? VLV_L3SQCREG1_SQGHPCI_DEFAULT :
1746 IVB_L3SQCREG1_SQGHPCI_DEFAULT;
1747
1748 anv_pack_struct(&l3cr2, GENX(L3CNTLREG2),
1749 .SLMEnable = has_slm,
1750 .URBLowBandwidth = urb_low_bw,
1751 .URBAllocation = cfg->n[GEN_L3P_URB] - n0_urb,
1752 #if !GEN_IS_HASWELL
1753 .ALLAllocation = cfg->n[GEN_L3P_ALL],
1754 #endif
1755 .ROAllocation = cfg->n[GEN_L3P_RO],
1756 .DCAllocation = cfg->n[GEN_L3P_DC]);
1757
1758 anv_pack_struct(&l3cr3, GENX(L3CNTLREG3),
1759 .ISAllocation = cfg->n[GEN_L3P_IS],
1760 .ISLowBandwidth = 0,
1761 .CAllocation = cfg->n[GEN_L3P_C],
1762 .CLowBandwidth = 0,
1763 .TAllocation = cfg->n[GEN_L3P_T],
1764 .TLowBandwidth = 0);
1765
1766 /* Set up the L3 partitioning. */
1767 emit_lri(&cmd_buffer->batch, GENX(L3SQCREG1_num), l3sqcr1);
1768 emit_lri(&cmd_buffer->batch, GENX(L3CNTLREG2_num), l3cr2);
1769 emit_lri(&cmd_buffer->batch, GENX(L3CNTLREG3_num), l3cr3);
1770
1771 #if GEN_IS_HASWELL
1772 if (cmd_buffer->device->instance->physicalDevice.cmd_parser_version >= 4) {
1773 /* Enable L3 atomics on HSW if we have a DC partition, otherwise keep
1774 * them disabled to avoid crashing the system hard.
1775 */
1776 uint32_t scratch1, chicken3;
1777 anv_pack_struct(&scratch1, GENX(SCRATCH1),
1778 .L3AtomicDisable = !has_dc);
1779 anv_pack_struct(&chicken3, GENX(CHICKEN3),
1780 .L3AtomicDisableMask = true,
1781 .L3AtomicDisable = !has_dc);
1782 emit_lri(&cmd_buffer->batch, GENX(SCRATCH1_num), scratch1);
1783 emit_lri(&cmd_buffer->batch, GENX(CHICKEN3_num), chicken3);
1784 }
1785 #endif
1786
1787 #endif
1788
1789 cmd_buffer->state.current_l3_config = cfg;
1790 }
1791
1792 void
1793 genX(cmd_buffer_apply_pipe_flushes)(struct anv_cmd_buffer *cmd_buffer)
1794 {
1795 enum anv_pipe_bits bits = cmd_buffer->state.pending_pipe_bits;
1796
1797 /* Flushes are pipelined while invalidations are handled immediately.
1798 * Therefore, if we're flushing anything then we need to schedule a stall
1799 * before any invalidations can happen.
1800 */
1801 if (bits & ANV_PIPE_FLUSH_BITS)
1802 bits |= ANV_PIPE_NEEDS_CS_STALL_BIT;
1803
1804 /* If we're going to do an invalidate and we have a pending CS stall that
1805 * has yet to be resolved, we do the CS stall now.
1806 */
1807 if ((bits & ANV_PIPE_INVALIDATE_BITS) &&
1808 (bits & ANV_PIPE_NEEDS_CS_STALL_BIT)) {
1809 bits |= ANV_PIPE_CS_STALL_BIT;
1810 bits &= ~ANV_PIPE_NEEDS_CS_STALL_BIT;
1811 }
1812
1813 if (bits & (ANV_PIPE_FLUSH_BITS | ANV_PIPE_CS_STALL_BIT)) {
1814 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pipe) {
1815 pipe.DepthCacheFlushEnable = bits & ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
1816 pipe.DCFlushEnable = bits & ANV_PIPE_DATA_CACHE_FLUSH_BIT;
1817 pipe.RenderTargetCacheFlushEnable =
1818 bits & ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
1819
1820 pipe.DepthStallEnable = bits & ANV_PIPE_DEPTH_STALL_BIT;
1821 pipe.CommandStreamerStallEnable = bits & ANV_PIPE_CS_STALL_BIT;
1822 pipe.StallAtPixelScoreboard = bits & ANV_PIPE_STALL_AT_SCOREBOARD_BIT;
1823
1824 /*
1825 * According to the Broadwell documentation, any PIPE_CONTROL with the
1826 * "Command Streamer Stall" bit set must also have another bit set,
1827 * with five different options:
1828 *
1829 * - Render Target Cache Flush
1830 * - Depth Cache Flush
1831 * - Stall at Pixel Scoreboard
1832 * - Post-Sync Operation
1833 * - Depth Stall
1834 * - DC Flush Enable
1835 *
1836 * I chose "Stall at Pixel Scoreboard" since that's what we use in
1837 * mesa and it seems to work fine. The choice is fairly arbitrary.
1838 */
1839 if ((bits & ANV_PIPE_CS_STALL_BIT) &&
1840 !(bits & (ANV_PIPE_FLUSH_BITS | ANV_PIPE_DEPTH_STALL_BIT |
1841 ANV_PIPE_STALL_AT_SCOREBOARD_BIT)))
1842 pipe.StallAtPixelScoreboard = true;
1843 }
1844
1845 /* If a render target flush was emitted, then we can toggle off the bit
1846 * saying that render target writes are ongoing.
1847 */
1848 if (bits & ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT)
1849 bits &= ~(ANV_PIPE_RENDER_TARGET_BUFFER_WRITES);
1850
1851 bits &= ~(ANV_PIPE_FLUSH_BITS | ANV_PIPE_CS_STALL_BIT);
1852 }
1853
1854 if (bits & ANV_PIPE_INVALIDATE_BITS) {
1855 /* From the SKL PRM, Vol. 2a, "PIPE_CONTROL",
1856 *
1857 * "If the VF Cache Invalidation Enable is set to a 1 in a
1858 * PIPE_CONTROL, a separate Null PIPE_CONTROL, all bitfields sets to
1859 * 0, with the VF Cache Invalidation Enable set to 0 needs to be sent
1860 * prior to the PIPE_CONTROL with VF Cache Invalidation Enable set to
1861 * a 1."
1862 *
1863 * This appears to hang Broadwell, so we restrict it to just gen9.
1864 */
1865 if (GEN_GEN == 9 && (bits & ANV_PIPE_VF_CACHE_INVALIDATE_BIT))
1866 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pipe);
1867
1868 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pipe) {
1869 pipe.StateCacheInvalidationEnable =
1870 bits & ANV_PIPE_STATE_CACHE_INVALIDATE_BIT;
1871 pipe.ConstantCacheInvalidationEnable =
1872 bits & ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
1873 pipe.VFCacheInvalidationEnable =
1874 bits & ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
1875 pipe.TextureCacheInvalidationEnable =
1876 bits & ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
1877 pipe.InstructionCacheInvalidateEnable =
1878 bits & ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT;
1879
1880 /* From the SKL PRM, Vol. 2a, "PIPE_CONTROL",
1881 *
1882 * "When VF Cache Invalidate is set “Post Sync Operation” must be
1883 * enabled to “Write Immediate Data” or “Write PS Depth Count” or
1884 * “Write Timestamp”.
1885 */
1886 if (GEN_GEN == 9 && pipe.VFCacheInvalidationEnable) {
1887 pipe.PostSyncOperation = WriteImmediateData;
1888 pipe.Address =
1889 (struct anv_address) { &cmd_buffer->device->workaround_bo, 0 };
1890 }
1891 }
1892
1893 bits &= ~ANV_PIPE_INVALIDATE_BITS;
1894 }
1895
1896 cmd_buffer->state.pending_pipe_bits = bits;
1897 }
1898
1899 void genX(CmdPipelineBarrier)(
1900 VkCommandBuffer commandBuffer,
1901 VkPipelineStageFlags srcStageMask,
1902 VkPipelineStageFlags destStageMask,
1903 VkBool32 byRegion,
1904 uint32_t memoryBarrierCount,
1905 const VkMemoryBarrier* pMemoryBarriers,
1906 uint32_t bufferMemoryBarrierCount,
1907 const VkBufferMemoryBarrier* pBufferMemoryBarriers,
1908 uint32_t imageMemoryBarrierCount,
1909 const VkImageMemoryBarrier* pImageMemoryBarriers)
1910 {
1911 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1912
1913 /* XXX: Right now, we're really dumb and just flush whatever categories
1914 * the app asks for. One of these days we may make this a bit better
1915 * but right now that's all the hardware allows for in most areas.
1916 */
1917 VkAccessFlags src_flags = 0;
1918 VkAccessFlags dst_flags = 0;
1919
1920 for (uint32_t i = 0; i < memoryBarrierCount; i++) {
1921 src_flags |= pMemoryBarriers[i].srcAccessMask;
1922 dst_flags |= pMemoryBarriers[i].dstAccessMask;
1923 }
1924
1925 for (uint32_t i = 0; i < bufferMemoryBarrierCount; i++) {
1926 src_flags |= pBufferMemoryBarriers[i].srcAccessMask;
1927 dst_flags |= pBufferMemoryBarriers[i].dstAccessMask;
1928 }
1929
1930 for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) {
1931 src_flags |= pImageMemoryBarriers[i].srcAccessMask;
1932 dst_flags |= pImageMemoryBarriers[i].dstAccessMask;
1933 ANV_FROM_HANDLE(anv_image, image, pImageMemoryBarriers[i].image);
1934 const VkImageSubresourceRange *range =
1935 &pImageMemoryBarriers[i].subresourceRange;
1936
1937 uint32_t base_layer, layer_count;
1938 if (image->type == VK_IMAGE_TYPE_3D) {
1939 base_layer = 0;
1940 layer_count = anv_minify(image->extent.depth, range->baseMipLevel);
1941 } else {
1942 base_layer = range->baseArrayLayer;
1943 layer_count = anv_get_layerCount(image, range);
1944 }
1945
1946 if (range->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT) {
1947 transition_depth_buffer(cmd_buffer, image,
1948 pImageMemoryBarriers[i].oldLayout,
1949 pImageMemoryBarriers[i].newLayout);
1950 }
1951
1952 if (range->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT) {
1953 transition_stencil_buffer(cmd_buffer, image,
1954 range->baseMipLevel,
1955 anv_get_levelCount(image, range),
1956 base_layer, layer_count,
1957 pImageMemoryBarriers[i].oldLayout,
1958 pImageMemoryBarriers[i].newLayout);
1959 }
1960
1961 if (range->aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
1962 VkImageAspectFlags color_aspects =
1963 anv_image_expand_aspects(image, range->aspectMask);
1964 uint32_t aspect_bit;
1965 anv_foreach_image_aspect_bit(aspect_bit, image, color_aspects) {
1966 transition_color_buffer(cmd_buffer, image, 1UL << aspect_bit,
1967 range->baseMipLevel,
1968 anv_get_levelCount(image, range),
1969 base_layer, layer_count,
1970 pImageMemoryBarriers[i].oldLayout,
1971 pImageMemoryBarriers[i].newLayout);
1972 }
1973 }
1974 }
1975
1976 cmd_buffer->state.pending_pipe_bits |=
1977 anv_pipe_flush_bits_for_access_flags(src_flags) |
1978 anv_pipe_invalidate_bits_for_access_flags(dst_flags);
1979 }
1980
1981 static void
1982 cmd_buffer_alloc_push_constants(struct anv_cmd_buffer *cmd_buffer)
1983 {
1984 VkShaderStageFlags stages =
1985 cmd_buffer->state.gfx.base.pipeline->active_stages;
1986
1987 /* In order to avoid thrash, we assume that vertex and fragment stages
1988 * always exist. In the rare case where one is missing *and* the other
1989 * uses push concstants, this may be suboptimal. However, avoiding stalls
1990 * seems more important.
1991 */
1992 stages |= VK_SHADER_STAGE_FRAGMENT_BIT | VK_SHADER_STAGE_VERTEX_BIT;
1993
1994 if (stages == cmd_buffer->state.push_constant_stages)
1995 return;
1996
1997 #if GEN_GEN >= 8
1998 const unsigned push_constant_kb = 32;
1999 #elif GEN_IS_HASWELL
2000 const unsigned push_constant_kb = cmd_buffer->device->info.gt == 3 ? 32 : 16;
2001 #else
2002 const unsigned push_constant_kb = 16;
2003 #endif
2004
2005 const unsigned num_stages =
2006 util_bitcount(stages & VK_SHADER_STAGE_ALL_GRAPHICS);
2007 unsigned size_per_stage = push_constant_kb / num_stages;
2008
2009 /* Broadwell+ and Haswell gt3 require that the push constant sizes be in
2010 * units of 2KB. Incidentally, these are the same platforms that have
2011 * 32KB worth of push constant space.
2012 */
2013 if (push_constant_kb == 32)
2014 size_per_stage &= ~1u;
2015
2016 uint32_t kb_used = 0;
2017 for (int i = MESA_SHADER_VERTEX; i < MESA_SHADER_FRAGMENT; i++) {
2018 unsigned push_size = (stages & (1 << i)) ? size_per_stage : 0;
2019 anv_batch_emit(&cmd_buffer->batch,
2020 GENX(3DSTATE_PUSH_CONSTANT_ALLOC_VS), alloc) {
2021 alloc._3DCommandSubOpcode = 18 + i;
2022 alloc.ConstantBufferOffset = (push_size > 0) ? kb_used : 0;
2023 alloc.ConstantBufferSize = push_size;
2024 }
2025 kb_used += push_size;
2026 }
2027
2028 anv_batch_emit(&cmd_buffer->batch,
2029 GENX(3DSTATE_PUSH_CONSTANT_ALLOC_PS), alloc) {
2030 alloc.ConstantBufferOffset = kb_used;
2031 alloc.ConstantBufferSize = push_constant_kb - kb_used;
2032 }
2033
2034 cmd_buffer->state.push_constant_stages = stages;
2035
2036 /* From the BDW PRM for 3DSTATE_PUSH_CONSTANT_ALLOC_VS:
2037 *
2038 * "The 3DSTATE_CONSTANT_VS must be reprogrammed prior to
2039 * the next 3DPRIMITIVE command after programming the
2040 * 3DSTATE_PUSH_CONSTANT_ALLOC_VS"
2041 *
2042 * Since 3DSTATE_PUSH_CONSTANT_ALLOC_VS is programmed as part of
2043 * pipeline setup, we need to dirty push constants.
2044 */
2045 cmd_buffer->state.push_constants_dirty |= VK_SHADER_STAGE_ALL_GRAPHICS;
2046 }
2047
2048 static const struct anv_descriptor *
2049 anv_descriptor_for_binding(const struct anv_cmd_pipeline_state *pipe_state,
2050 const struct anv_pipeline_binding *binding)
2051 {
2052 assert(binding->set < MAX_SETS);
2053 const struct anv_descriptor_set *set =
2054 pipe_state->descriptors[binding->set];
2055 const uint32_t offset =
2056 set->layout->binding[binding->binding].descriptor_index;
2057 return &set->descriptors[offset + binding->index];
2058 }
2059
2060 static uint32_t
2061 dynamic_offset_for_binding(const struct anv_cmd_pipeline_state *pipe_state,
2062 const struct anv_pipeline_binding *binding)
2063 {
2064 assert(binding->set < MAX_SETS);
2065 const struct anv_descriptor_set *set =
2066 pipe_state->descriptors[binding->set];
2067
2068 uint32_t dynamic_offset_idx =
2069 pipe_state->layout->set[binding->set].dynamic_offset_start +
2070 set->layout->binding[binding->binding].dynamic_offset_index +
2071 binding->index;
2072
2073 return pipe_state->dynamic_offsets[dynamic_offset_idx];
2074 }
2075
2076 static struct anv_address
2077 anv_descriptor_set_address(struct anv_cmd_buffer *cmd_buffer,
2078 struct anv_descriptor_set *set)
2079 {
2080 if (set->pool) {
2081 /* This is a normal descriptor set */
2082 return (struct anv_address) {
2083 .bo = &set->pool->bo,
2084 .offset = set->desc_mem.offset,
2085 };
2086 } else {
2087 /* This is a push descriptor set. We have to flag it as used on the GPU
2088 * so that the next time we push descriptors, we grab a new memory.
2089 */
2090 struct anv_push_descriptor_set *push_set =
2091 (struct anv_push_descriptor_set *)set;
2092 push_set->set_used_on_gpu = true;
2093
2094 return (struct anv_address) {
2095 .bo = cmd_buffer->dynamic_state_stream.state_pool->block_pool.bo,
2096 .offset = set->desc_mem.offset,
2097 };
2098 }
2099 }
2100
2101 static VkResult
2102 emit_binding_table(struct anv_cmd_buffer *cmd_buffer,
2103 gl_shader_stage stage,
2104 struct anv_state *bt_state)
2105 {
2106 struct anv_subpass *subpass = cmd_buffer->state.subpass;
2107 struct anv_cmd_pipeline_state *pipe_state;
2108 struct anv_pipeline *pipeline;
2109 uint32_t state_offset;
2110
2111 switch (stage) {
2112 case MESA_SHADER_COMPUTE:
2113 pipe_state = &cmd_buffer->state.compute.base;
2114 break;
2115 default:
2116 pipe_state = &cmd_buffer->state.gfx.base;
2117 break;
2118 }
2119 pipeline = pipe_state->pipeline;
2120
2121 if (!anv_pipeline_has_stage(pipeline, stage)) {
2122 *bt_state = (struct anv_state) { 0, };
2123 return VK_SUCCESS;
2124 }
2125
2126 struct anv_pipeline_bind_map *map = &pipeline->shaders[stage]->bind_map;
2127 if (map->surface_count == 0) {
2128 *bt_state = (struct anv_state) { 0, };
2129 return VK_SUCCESS;
2130 }
2131
2132 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer,
2133 map->surface_count,
2134 &state_offset);
2135 uint32_t *bt_map = bt_state->map;
2136
2137 if (bt_state->map == NULL)
2138 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
2139
2140 /* We only need to emit relocs if we're not using softpin. If we are using
2141 * softpin then we always keep all user-allocated memory objects resident.
2142 */
2143 const bool need_client_mem_relocs =
2144 !cmd_buffer->device->instance->physicalDevice.use_softpin;
2145
2146 for (uint32_t s = 0; s < map->surface_count; s++) {
2147 struct anv_pipeline_binding *binding = &map->surface_to_descriptor[s];
2148
2149 struct anv_state surface_state;
2150
2151 if (binding->set == ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS) {
2152 /* Color attachment binding */
2153 assert(stage == MESA_SHADER_FRAGMENT);
2154 assert(binding->binding == 0);
2155 if (binding->index < subpass->color_count) {
2156 const unsigned att =
2157 subpass->color_attachments[binding->index].attachment;
2158
2159 /* From the Vulkan 1.0.46 spec:
2160 *
2161 * "If any color or depth/stencil attachments are
2162 * VK_ATTACHMENT_UNUSED, then no writes occur for those
2163 * attachments."
2164 */
2165 if (att == VK_ATTACHMENT_UNUSED) {
2166 surface_state = cmd_buffer->state.null_surface_state;
2167 } else {
2168 surface_state = cmd_buffer->state.attachments[att].color.state;
2169 }
2170 } else {
2171 surface_state = cmd_buffer->state.null_surface_state;
2172 }
2173
2174 bt_map[s] = surface_state.offset + state_offset;
2175 continue;
2176 } else if (binding->set == ANV_DESCRIPTOR_SET_SHADER_CONSTANTS) {
2177 struct anv_state surface_state =
2178 anv_cmd_buffer_alloc_surface_state(cmd_buffer);
2179
2180 struct anv_address constant_data = {
2181 .bo = pipeline->device->dynamic_state_pool.block_pool.bo,
2182 .offset = pipeline->shaders[stage]->constant_data.offset,
2183 };
2184 unsigned constant_data_size =
2185 pipeline->shaders[stage]->constant_data_size;
2186
2187 const enum isl_format format =
2188 anv_isl_format_for_descriptor_type(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER);
2189 anv_fill_buffer_surface_state(cmd_buffer->device,
2190 surface_state, format,
2191 constant_data, constant_data_size, 1);
2192
2193 bt_map[s] = surface_state.offset + state_offset;
2194 add_surface_reloc(cmd_buffer, surface_state, constant_data);
2195 continue;
2196 } else if (binding->set == ANV_DESCRIPTOR_SET_NUM_WORK_GROUPS) {
2197 /* This is always the first binding for compute shaders */
2198 assert(stage == MESA_SHADER_COMPUTE && s == 0);
2199 if (!get_cs_prog_data(pipeline)->uses_num_work_groups)
2200 continue;
2201
2202 struct anv_state surface_state =
2203 anv_cmd_buffer_alloc_surface_state(cmd_buffer);
2204
2205 const enum isl_format format =
2206 anv_isl_format_for_descriptor_type(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER);
2207 anv_fill_buffer_surface_state(cmd_buffer->device, surface_state,
2208 format,
2209 cmd_buffer->state.compute.num_workgroups,
2210 12, 1);
2211 bt_map[s] = surface_state.offset + state_offset;
2212 if (need_client_mem_relocs) {
2213 add_surface_reloc(cmd_buffer, surface_state,
2214 cmd_buffer->state.compute.num_workgroups);
2215 }
2216 continue;
2217 } else if (binding->set == ANV_DESCRIPTOR_SET_DESCRIPTORS) {
2218 /* This is a descriptor set buffer so the set index is actually
2219 * given by binding->binding. (Yes, that's confusing.)
2220 */
2221 struct anv_descriptor_set *set =
2222 pipe_state->descriptors[binding->binding];
2223 assert(set->desc_mem.alloc_size);
2224 assert(set->desc_surface_state.alloc_size);
2225 bt_map[s] = set->desc_surface_state.offset + state_offset;
2226 add_surface_reloc(cmd_buffer, set->desc_surface_state,
2227 anv_descriptor_set_address(cmd_buffer, set));
2228 continue;
2229 }
2230
2231 const struct anv_descriptor *desc =
2232 anv_descriptor_for_binding(pipe_state, binding);
2233
2234 switch (desc->type) {
2235 case VK_DESCRIPTOR_TYPE_SAMPLER:
2236 /* Nothing for us to do here */
2237 continue;
2238
2239 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
2240 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: {
2241 struct anv_surface_state sstate =
2242 (desc->layout == VK_IMAGE_LAYOUT_GENERAL) ?
2243 desc->image_view->planes[binding->plane].general_sampler_surface_state :
2244 desc->image_view->planes[binding->plane].optimal_sampler_surface_state;
2245 surface_state = sstate.state;
2246 assert(surface_state.alloc_size);
2247 if (need_client_mem_relocs)
2248 add_surface_state_relocs(cmd_buffer, sstate);
2249 break;
2250 }
2251 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
2252 assert(stage == MESA_SHADER_FRAGMENT);
2253 if ((desc->image_view->aspect_mask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) == 0) {
2254 /* For depth and stencil input attachments, we treat it like any
2255 * old texture that a user may have bound.
2256 */
2257 struct anv_surface_state sstate =
2258 (desc->layout == VK_IMAGE_LAYOUT_GENERAL) ?
2259 desc->image_view->planes[binding->plane].general_sampler_surface_state :
2260 desc->image_view->planes[binding->plane].optimal_sampler_surface_state;
2261 surface_state = sstate.state;
2262 assert(surface_state.alloc_size);
2263 if (need_client_mem_relocs)
2264 add_surface_state_relocs(cmd_buffer, sstate);
2265 } else {
2266 /* For color input attachments, we create the surface state at
2267 * vkBeginRenderPass time so that we can include aux and clear
2268 * color information.
2269 */
2270 assert(binding->input_attachment_index < subpass->input_count);
2271 const unsigned subpass_att = binding->input_attachment_index;
2272 const unsigned att = subpass->input_attachments[subpass_att].attachment;
2273 surface_state = cmd_buffer->state.attachments[att].input.state;
2274 }
2275 break;
2276
2277 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: {
2278 struct anv_surface_state sstate = (binding->write_only)
2279 ? desc->image_view->planes[binding->plane].writeonly_storage_surface_state
2280 : desc->image_view->planes[binding->plane].storage_surface_state;
2281 surface_state = sstate.state;
2282 assert(surface_state.alloc_size);
2283 if (need_client_mem_relocs)
2284 add_surface_state_relocs(cmd_buffer, sstate);
2285 break;
2286 }
2287
2288 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
2289 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
2290 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
2291 surface_state = desc->buffer_view->surface_state;
2292 assert(surface_state.alloc_size);
2293 if (need_client_mem_relocs) {
2294 add_surface_reloc(cmd_buffer, surface_state,
2295 desc->buffer_view->address);
2296 }
2297 break;
2298
2299 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
2300 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
2301 /* Compute the offset within the buffer */
2302 uint32_t dynamic_offset =
2303 dynamic_offset_for_binding(pipe_state, binding);
2304 uint64_t offset = desc->offset + dynamic_offset;
2305 /* Clamp to the buffer size */
2306 offset = MIN2(offset, desc->buffer->size);
2307 /* Clamp the range to the buffer size */
2308 uint32_t range = MIN2(desc->range, desc->buffer->size - offset);
2309
2310 struct anv_address address =
2311 anv_address_add(desc->buffer->address, offset);
2312
2313 surface_state =
2314 anv_state_stream_alloc(&cmd_buffer->surface_state_stream, 64, 64);
2315 enum isl_format format =
2316 anv_isl_format_for_descriptor_type(desc->type);
2317
2318 anv_fill_buffer_surface_state(cmd_buffer->device, surface_state,
2319 format, address, range, 1);
2320 if (need_client_mem_relocs)
2321 add_surface_reloc(cmd_buffer, surface_state, address);
2322 break;
2323 }
2324
2325 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
2326 surface_state = (binding->write_only)
2327 ? desc->buffer_view->writeonly_storage_surface_state
2328 : desc->buffer_view->storage_surface_state;
2329 assert(surface_state.alloc_size);
2330 if (need_client_mem_relocs) {
2331 add_surface_reloc(cmd_buffer, surface_state,
2332 desc->buffer_view->address);
2333 }
2334 break;
2335
2336 default:
2337 assert(!"Invalid descriptor type");
2338 continue;
2339 }
2340
2341 bt_map[s] = surface_state.offset + state_offset;
2342 }
2343
2344 #if GEN_GEN >= 11
2345 /* The PIPE_CONTROL command description says:
2346 *
2347 * "Whenever a Binding Table Index (BTI) used by a Render Taget Message
2348 * points to a different RENDER_SURFACE_STATE, SW must issue a Render
2349 * Target Cache Flush by enabling this bit. When render target flush
2350 * is set due to new association of BTI, PS Scoreboard Stall bit must
2351 * be set in this packet."
2352 *
2353 * FINISHME: Currently we shuffle around the surface states in the binding
2354 * table based on if they are getting used or not. So, we've to do below
2355 * pipe control flush for every binding table upload. Make changes so
2356 * that we do it only when we modify render target surface states.
2357 */
2358 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
2359 pc.RenderTargetCacheFlushEnable = true;
2360 pc.StallAtPixelScoreboard = true;
2361 }
2362 #endif
2363
2364 return VK_SUCCESS;
2365 }
2366
2367 static VkResult
2368 emit_samplers(struct anv_cmd_buffer *cmd_buffer,
2369 gl_shader_stage stage,
2370 struct anv_state *state)
2371 {
2372 struct anv_cmd_pipeline_state *pipe_state =
2373 stage == MESA_SHADER_COMPUTE ? &cmd_buffer->state.compute.base :
2374 &cmd_buffer->state.gfx.base;
2375 struct anv_pipeline *pipeline = pipe_state->pipeline;
2376
2377 if (!anv_pipeline_has_stage(pipeline, stage)) {
2378 *state = (struct anv_state) { 0, };
2379 return VK_SUCCESS;
2380 }
2381
2382 struct anv_pipeline_bind_map *map = &pipeline->shaders[stage]->bind_map;
2383 if (map->sampler_count == 0) {
2384 *state = (struct anv_state) { 0, };
2385 return VK_SUCCESS;
2386 }
2387
2388 uint32_t size = map->sampler_count * 16;
2389 *state = anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, size, 32);
2390
2391 if (state->map == NULL)
2392 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
2393
2394 for (uint32_t s = 0; s < map->sampler_count; s++) {
2395 struct anv_pipeline_binding *binding = &map->sampler_to_descriptor[s];
2396 const struct anv_descriptor *desc =
2397 anv_descriptor_for_binding(pipe_state, binding);
2398
2399 if (desc->type != VK_DESCRIPTOR_TYPE_SAMPLER &&
2400 desc->type != VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
2401 continue;
2402
2403 struct anv_sampler *sampler = desc->sampler;
2404
2405 /* This can happen if we have an unfilled slot since TYPE_SAMPLER
2406 * happens to be zero.
2407 */
2408 if (sampler == NULL)
2409 continue;
2410
2411 memcpy(state->map + (s * 16),
2412 sampler->state[binding->plane], sizeof(sampler->state[0]));
2413 }
2414
2415 return VK_SUCCESS;
2416 }
2417
2418 static uint32_t
2419 flush_descriptor_sets(struct anv_cmd_buffer *cmd_buffer)
2420 {
2421 struct anv_pipeline *pipeline = cmd_buffer->state.gfx.base.pipeline;
2422
2423 VkShaderStageFlags dirty = cmd_buffer->state.descriptors_dirty &
2424 pipeline->active_stages;
2425
2426 VkResult result = VK_SUCCESS;
2427 anv_foreach_stage(s, dirty) {
2428 result = emit_samplers(cmd_buffer, s, &cmd_buffer->state.samplers[s]);
2429 if (result != VK_SUCCESS)
2430 break;
2431 result = emit_binding_table(cmd_buffer, s,
2432 &cmd_buffer->state.binding_tables[s]);
2433 if (result != VK_SUCCESS)
2434 break;
2435 }
2436
2437 if (result != VK_SUCCESS) {
2438 assert(result == VK_ERROR_OUT_OF_DEVICE_MEMORY);
2439
2440 result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
2441 if (result != VK_SUCCESS)
2442 return 0;
2443
2444 /* Re-emit state base addresses so we get the new surface state base
2445 * address before we start emitting binding tables etc.
2446 */
2447 genX(cmd_buffer_emit_state_base_address)(cmd_buffer);
2448
2449 /* Re-emit all active binding tables */
2450 dirty |= pipeline->active_stages;
2451 anv_foreach_stage(s, dirty) {
2452 result = emit_samplers(cmd_buffer, s, &cmd_buffer->state.samplers[s]);
2453 if (result != VK_SUCCESS) {
2454 anv_batch_set_error(&cmd_buffer->batch, result);
2455 return 0;
2456 }
2457 result = emit_binding_table(cmd_buffer, s,
2458 &cmd_buffer->state.binding_tables[s]);
2459 if (result != VK_SUCCESS) {
2460 anv_batch_set_error(&cmd_buffer->batch, result);
2461 return 0;
2462 }
2463 }
2464 }
2465
2466 cmd_buffer->state.descriptors_dirty &= ~dirty;
2467
2468 return dirty;
2469 }
2470
2471 static void
2472 cmd_buffer_emit_descriptor_pointers(struct anv_cmd_buffer *cmd_buffer,
2473 uint32_t stages)
2474 {
2475 static const uint32_t sampler_state_opcodes[] = {
2476 [MESA_SHADER_VERTEX] = 43,
2477 [MESA_SHADER_TESS_CTRL] = 44, /* HS */
2478 [MESA_SHADER_TESS_EVAL] = 45, /* DS */
2479 [MESA_SHADER_GEOMETRY] = 46,
2480 [MESA_SHADER_FRAGMENT] = 47,
2481 [MESA_SHADER_COMPUTE] = 0,
2482 };
2483
2484 static const uint32_t binding_table_opcodes[] = {
2485 [MESA_SHADER_VERTEX] = 38,
2486 [MESA_SHADER_TESS_CTRL] = 39,
2487 [MESA_SHADER_TESS_EVAL] = 40,
2488 [MESA_SHADER_GEOMETRY] = 41,
2489 [MESA_SHADER_FRAGMENT] = 42,
2490 [MESA_SHADER_COMPUTE] = 0,
2491 };
2492
2493 anv_foreach_stage(s, stages) {
2494 assert(s < ARRAY_SIZE(binding_table_opcodes));
2495 assert(binding_table_opcodes[s] > 0);
2496
2497 if (cmd_buffer->state.samplers[s].alloc_size > 0) {
2498 anv_batch_emit(&cmd_buffer->batch,
2499 GENX(3DSTATE_SAMPLER_STATE_POINTERS_VS), ssp) {
2500 ssp._3DCommandSubOpcode = sampler_state_opcodes[s];
2501 ssp.PointertoVSSamplerState = cmd_buffer->state.samplers[s].offset;
2502 }
2503 }
2504
2505 /* Always emit binding table pointers if we're asked to, since on SKL
2506 * this is what flushes push constants. */
2507 anv_batch_emit(&cmd_buffer->batch,
2508 GENX(3DSTATE_BINDING_TABLE_POINTERS_VS), btp) {
2509 btp._3DCommandSubOpcode = binding_table_opcodes[s];
2510 btp.PointertoVSBindingTable = cmd_buffer->state.binding_tables[s].offset;
2511 }
2512 }
2513 }
2514
2515 static void
2516 cmd_buffer_flush_push_constants(struct anv_cmd_buffer *cmd_buffer,
2517 VkShaderStageFlags dirty_stages)
2518 {
2519 const struct anv_cmd_graphics_state *gfx_state = &cmd_buffer->state.gfx;
2520 const struct anv_pipeline *pipeline = gfx_state->base.pipeline;
2521
2522 static const uint32_t push_constant_opcodes[] = {
2523 [MESA_SHADER_VERTEX] = 21,
2524 [MESA_SHADER_TESS_CTRL] = 25, /* HS */
2525 [MESA_SHADER_TESS_EVAL] = 26, /* DS */
2526 [MESA_SHADER_GEOMETRY] = 22,
2527 [MESA_SHADER_FRAGMENT] = 23,
2528 [MESA_SHADER_COMPUTE] = 0,
2529 };
2530
2531 VkShaderStageFlags flushed = 0;
2532
2533 anv_foreach_stage(stage, dirty_stages) {
2534 assert(stage < ARRAY_SIZE(push_constant_opcodes));
2535 assert(push_constant_opcodes[stage] > 0);
2536
2537 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CONSTANT_VS), c) {
2538 c._3DCommandSubOpcode = push_constant_opcodes[stage];
2539
2540 if (anv_pipeline_has_stage(pipeline, stage)) {
2541 #if GEN_GEN >= 8 || GEN_IS_HASWELL
2542 const struct brw_stage_prog_data *prog_data =
2543 pipeline->shaders[stage]->prog_data;
2544 const struct anv_pipeline_bind_map *bind_map =
2545 &pipeline->shaders[stage]->bind_map;
2546
2547 /* The Skylake PRM contains the following restriction:
2548 *
2549 * "The driver must ensure The following case does not occur
2550 * without a flush to the 3D engine: 3DSTATE_CONSTANT_* with
2551 * buffer 3 read length equal to zero committed followed by a
2552 * 3DSTATE_CONSTANT_* with buffer 0 read length not equal to
2553 * zero committed."
2554 *
2555 * To avoid this, we program the buffers in the highest slots.
2556 * This way, slot 0 is only used if slot 3 is also used.
2557 */
2558 int n = 3;
2559
2560 for (int i = 3; i >= 0; i--) {
2561 const struct brw_ubo_range *range = &prog_data->ubo_ranges[i];
2562 if (range->length == 0)
2563 continue;
2564
2565 const unsigned surface =
2566 prog_data->binding_table.ubo_start + range->block;
2567
2568 assert(surface <= bind_map->surface_count);
2569 const struct anv_pipeline_binding *binding =
2570 &bind_map->surface_to_descriptor[surface];
2571
2572 struct anv_address read_addr;
2573 uint32_t read_len;
2574 if (binding->set == ANV_DESCRIPTOR_SET_SHADER_CONSTANTS) {
2575 struct anv_address constant_data = {
2576 .bo = pipeline->device->dynamic_state_pool.block_pool.bo,
2577 .offset = pipeline->shaders[stage]->constant_data.offset,
2578 };
2579 unsigned constant_data_size =
2580 pipeline->shaders[stage]->constant_data_size;
2581
2582 read_len = MIN2(range->length,
2583 DIV_ROUND_UP(constant_data_size, 32) - range->start);
2584 read_addr = anv_address_add(constant_data,
2585 range->start * 32);
2586 } else if (binding->set == ANV_DESCRIPTOR_SET_DESCRIPTORS) {
2587 /* This is a descriptor set buffer so the set index is
2588 * actually given by binding->binding. (Yes, that's
2589 * confusing.)
2590 */
2591 struct anv_descriptor_set *set =
2592 gfx_state->base.descriptors[binding->binding];
2593 struct anv_address desc_buffer_addr =
2594 anv_descriptor_set_address(cmd_buffer, set);
2595 const unsigned desc_buffer_size = set->desc_mem.alloc_size;
2596
2597 read_len = MIN2(range->length,
2598 DIV_ROUND_UP(desc_buffer_size, 32) - range->start);
2599 read_addr = anv_address_add(desc_buffer_addr,
2600 range->start * 32);
2601 } else {
2602 const struct anv_descriptor *desc =
2603 anv_descriptor_for_binding(&gfx_state->base, binding);
2604
2605 if (desc->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER) {
2606 read_len = MIN2(range->length,
2607 DIV_ROUND_UP(desc->buffer_view->range, 32) - range->start);
2608 read_addr = anv_address_add(desc->buffer_view->address,
2609 range->start * 32);
2610 } else {
2611 assert(desc->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC);
2612
2613 uint32_t dynamic_offset =
2614 dynamic_offset_for_binding(&gfx_state->base, binding);
2615 uint32_t buf_offset =
2616 MIN2(desc->offset + dynamic_offset, desc->buffer->size);
2617 uint32_t buf_range =
2618 MIN2(desc->range, desc->buffer->size - buf_offset);
2619
2620 read_len = MIN2(range->length,
2621 DIV_ROUND_UP(buf_range, 32) - range->start);
2622 read_addr = anv_address_add(desc->buffer->address,
2623 buf_offset + range->start * 32);
2624 }
2625 }
2626
2627 if (read_len > 0) {
2628 c.ConstantBody.Buffer[n] = read_addr;
2629 c.ConstantBody.ReadLength[n] = read_len;
2630 n--;
2631 }
2632 }
2633
2634 struct anv_state state =
2635 anv_cmd_buffer_push_constants(cmd_buffer, stage);
2636
2637 if (state.alloc_size > 0) {
2638 c.ConstantBody.Buffer[n] = (struct anv_address) {
2639 .bo = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
2640 .offset = state.offset,
2641 };
2642 c.ConstantBody.ReadLength[n] =
2643 DIV_ROUND_UP(state.alloc_size, 32);
2644 }
2645 #else
2646 /* For Ivy Bridge, the push constants packets have a different
2647 * rule that would require us to iterate in the other direction
2648 * and possibly mess around with dynamic state base address.
2649 * Don't bother; just emit regular push constants at n = 0.
2650 */
2651 struct anv_state state =
2652 anv_cmd_buffer_push_constants(cmd_buffer, stage);
2653
2654 if (state.alloc_size > 0) {
2655 c.ConstantBody.Buffer[0].offset = state.offset,
2656 c.ConstantBody.ReadLength[0] =
2657 DIV_ROUND_UP(state.alloc_size, 32);
2658 }
2659 #endif
2660 }
2661 }
2662
2663 flushed |= mesa_to_vk_shader_stage(stage);
2664 }
2665
2666 cmd_buffer->state.push_constants_dirty &= ~flushed;
2667 }
2668
2669 void
2670 genX(cmd_buffer_flush_state)(struct anv_cmd_buffer *cmd_buffer)
2671 {
2672 struct anv_pipeline *pipeline = cmd_buffer->state.gfx.base.pipeline;
2673 uint32_t *p;
2674
2675 uint32_t vb_emit = cmd_buffer->state.gfx.vb_dirty & pipeline->vb_used;
2676 if (cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_PIPELINE)
2677 vb_emit |= pipeline->vb_used;
2678
2679 assert((pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT) == 0);
2680
2681 genX(cmd_buffer_config_l3)(cmd_buffer, pipeline->urb.l3_config);
2682
2683 genX(flush_pipeline_select_3d)(cmd_buffer);
2684
2685 if (vb_emit) {
2686 const uint32_t num_buffers = __builtin_popcount(vb_emit);
2687 const uint32_t num_dwords = 1 + num_buffers * 4;
2688
2689 p = anv_batch_emitn(&cmd_buffer->batch, num_dwords,
2690 GENX(3DSTATE_VERTEX_BUFFERS));
2691 uint32_t vb, i = 0;
2692 for_each_bit(vb, vb_emit) {
2693 struct anv_buffer *buffer = cmd_buffer->state.vertex_bindings[vb].buffer;
2694 uint32_t offset = cmd_buffer->state.vertex_bindings[vb].offset;
2695
2696 struct GENX(VERTEX_BUFFER_STATE) state = {
2697 .VertexBufferIndex = vb,
2698
2699 .MOCS = anv_mocs_for_bo(cmd_buffer->device, buffer->address.bo),
2700 #if GEN_GEN <= 7
2701 .BufferAccessType = pipeline->vb[vb].instanced ? INSTANCEDATA : VERTEXDATA,
2702 .InstanceDataStepRate = pipeline->vb[vb].instance_divisor,
2703 #endif
2704
2705 .AddressModifyEnable = true,
2706 .BufferPitch = pipeline->vb[vb].stride,
2707 .BufferStartingAddress = anv_address_add(buffer->address, offset),
2708
2709 #if GEN_GEN >= 8
2710 .BufferSize = buffer->size - offset
2711 #else
2712 .EndAddress = anv_address_add(buffer->address, buffer->size - 1),
2713 #endif
2714 };
2715
2716 GENX(VERTEX_BUFFER_STATE_pack)(&cmd_buffer->batch, &p[1 + i * 4], &state);
2717 i++;
2718 }
2719 }
2720
2721 cmd_buffer->state.gfx.vb_dirty &= ~vb_emit;
2722
2723 #if GEN_GEN >= 8
2724 if (cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_XFB_ENABLE) {
2725 /* We don't need any per-buffer dirty tracking because you're not
2726 * allowed to bind different XFB buffers while XFB is enabled.
2727 */
2728 for (unsigned idx = 0; idx < MAX_XFB_BUFFERS; idx++) {
2729 struct anv_xfb_binding *xfb = &cmd_buffer->state.xfb_bindings[idx];
2730 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_SO_BUFFER), sob) {
2731 sob.SOBufferIndex = idx;
2732
2733 if (cmd_buffer->state.xfb_enabled && xfb->buffer && xfb->size != 0) {
2734 sob.SOBufferEnable = true;
2735 sob.MOCS = cmd_buffer->device->default_mocs,
2736 sob.StreamOffsetWriteEnable = false;
2737 sob.SurfaceBaseAddress = anv_address_add(xfb->buffer->address,
2738 xfb->offset);
2739 /* Size is in DWords - 1 */
2740 sob.SurfaceSize = xfb->size / 4 - 1;
2741 }
2742 }
2743 }
2744
2745 /* CNL and later require a CS stall after 3DSTATE_SO_BUFFER */
2746 if (GEN_GEN >= 10)
2747 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_CS_STALL_BIT;
2748 }
2749 #endif
2750
2751 if (cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_PIPELINE) {
2752 anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
2753
2754 /* The exact descriptor layout is pulled from the pipeline, so we need
2755 * to re-emit binding tables on every pipeline change.
2756 */
2757 cmd_buffer->state.descriptors_dirty |= pipeline->active_stages;
2758
2759 /* If the pipeline changed, we may need to re-allocate push constant
2760 * space in the URB.
2761 */
2762 cmd_buffer_alloc_push_constants(cmd_buffer);
2763 }
2764
2765 #if GEN_GEN <= 7
2766 if (cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_VERTEX_BIT ||
2767 cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_VERTEX_BIT) {
2768 /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
2769 *
2770 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
2771 * stall needs to be sent just prior to any 3DSTATE_VS,
2772 * 3DSTATE_URB_VS, 3DSTATE_CONSTANT_VS,
2773 * 3DSTATE_BINDING_TABLE_POINTER_VS,
2774 * 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one
2775 * PIPE_CONTROL needs to be sent before any combination of VS
2776 * associated 3DSTATE."
2777 */
2778 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
2779 pc.DepthStallEnable = true;
2780 pc.PostSyncOperation = WriteImmediateData;
2781 pc.Address =
2782 (struct anv_address) { &cmd_buffer->device->workaround_bo, 0 };
2783 }
2784 }
2785 #endif
2786
2787 /* Render targets live in the same binding table as fragment descriptors */
2788 if (cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_RENDER_TARGETS)
2789 cmd_buffer->state.descriptors_dirty |= VK_SHADER_STAGE_FRAGMENT_BIT;
2790
2791 /* We emit the binding tables and sampler tables first, then emit push
2792 * constants and then finally emit binding table and sampler table
2793 * pointers. It has to happen in this order, since emitting the binding
2794 * tables may change the push constants (in case of storage images). After
2795 * emitting push constants, on SKL+ we have to emit the corresponding
2796 * 3DSTATE_BINDING_TABLE_POINTER_* for the push constants to take effect.
2797 */
2798 uint32_t dirty = 0;
2799 if (cmd_buffer->state.descriptors_dirty)
2800 dirty = flush_descriptor_sets(cmd_buffer);
2801
2802 if (dirty || cmd_buffer->state.push_constants_dirty) {
2803 /* Because we're pushing UBOs, we have to push whenever either
2804 * descriptors or push constants is dirty.
2805 */
2806 dirty |= cmd_buffer->state.push_constants_dirty;
2807 dirty &= ANV_STAGE_MASK & VK_SHADER_STAGE_ALL_GRAPHICS;
2808 cmd_buffer_flush_push_constants(cmd_buffer, dirty);
2809 }
2810
2811 if (dirty)
2812 cmd_buffer_emit_descriptor_pointers(cmd_buffer, dirty);
2813
2814 if (cmd_buffer->state.gfx.dirty & ANV_CMD_DIRTY_DYNAMIC_VIEWPORT)
2815 gen8_cmd_buffer_emit_viewport(cmd_buffer);
2816
2817 if (cmd_buffer->state.gfx.dirty & (ANV_CMD_DIRTY_DYNAMIC_VIEWPORT |
2818 ANV_CMD_DIRTY_PIPELINE)) {
2819 gen8_cmd_buffer_emit_depth_viewport(cmd_buffer,
2820 pipeline->depth_clamp_enable);
2821 }
2822
2823 if (cmd_buffer->state.gfx.dirty & (ANV_CMD_DIRTY_DYNAMIC_SCISSOR |
2824 ANV_CMD_DIRTY_RENDER_TARGETS))
2825 gen7_cmd_buffer_emit_scissor(cmd_buffer);
2826
2827 genX(cmd_buffer_flush_dynamic_state)(cmd_buffer);
2828
2829 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
2830 }
2831
2832 static void
2833 emit_vertex_bo(struct anv_cmd_buffer *cmd_buffer,
2834 struct anv_address addr,
2835 uint32_t size, uint32_t index)
2836 {
2837 uint32_t *p = anv_batch_emitn(&cmd_buffer->batch, 5,
2838 GENX(3DSTATE_VERTEX_BUFFERS));
2839
2840 GENX(VERTEX_BUFFER_STATE_pack)(&cmd_buffer->batch, p + 1,
2841 &(struct GENX(VERTEX_BUFFER_STATE)) {
2842 .VertexBufferIndex = index,
2843 .AddressModifyEnable = true,
2844 .BufferPitch = 0,
2845 .MOCS = anv_mocs_for_bo(cmd_buffer->device, addr.bo),
2846 #if (GEN_GEN >= 8)
2847 .BufferStartingAddress = addr,
2848 .BufferSize = size
2849 #else
2850 .BufferStartingAddress = addr,
2851 .EndAddress = anv_address_add(addr, size),
2852 #endif
2853 });
2854 }
2855
2856 static void
2857 emit_base_vertex_instance_bo(struct anv_cmd_buffer *cmd_buffer,
2858 struct anv_address addr)
2859 {
2860 emit_vertex_bo(cmd_buffer, addr, 8, ANV_SVGS_VB_INDEX);
2861 }
2862
2863 static void
2864 emit_base_vertex_instance(struct anv_cmd_buffer *cmd_buffer,
2865 uint32_t base_vertex, uint32_t base_instance)
2866 {
2867 struct anv_state id_state =
2868 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, 8, 4);
2869
2870 ((uint32_t *)id_state.map)[0] = base_vertex;
2871 ((uint32_t *)id_state.map)[1] = base_instance;
2872
2873 struct anv_address addr = {
2874 .bo = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
2875 .offset = id_state.offset,
2876 };
2877
2878 emit_base_vertex_instance_bo(cmd_buffer, addr);
2879 }
2880
2881 static void
2882 emit_draw_index(struct anv_cmd_buffer *cmd_buffer, uint32_t draw_index)
2883 {
2884 struct anv_state state =
2885 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, 4, 4);
2886
2887 ((uint32_t *)state.map)[0] = draw_index;
2888
2889 struct anv_address addr = {
2890 .bo = cmd_buffer->device->dynamic_state_pool.block_pool.bo,
2891 .offset = state.offset,
2892 };
2893
2894 emit_vertex_bo(cmd_buffer, addr, 4, ANV_DRAWID_VB_INDEX);
2895 }
2896
2897 void genX(CmdDraw)(
2898 VkCommandBuffer commandBuffer,
2899 uint32_t vertexCount,
2900 uint32_t instanceCount,
2901 uint32_t firstVertex,
2902 uint32_t firstInstance)
2903 {
2904 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
2905 struct anv_pipeline *pipeline = cmd_buffer->state.gfx.base.pipeline;
2906 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
2907
2908 if (anv_batch_has_error(&cmd_buffer->batch))
2909 return;
2910
2911 genX(cmd_buffer_flush_state)(cmd_buffer);
2912
2913 if (cmd_buffer->state.conditional_render_enabled)
2914 genX(cmd_emit_conditional_render_predicate)(cmd_buffer);
2915
2916 if (vs_prog_data->uses_firstvertex ||
2917 vs_prog_data->uses_baseinstance)
2918 emit_base_vertex_instance(cmd_buffer, firstVertex, firstInstance);
2919 if (vs_prog_data->uses_drawid)
2920 emit_draw_index(cmd_buffer, 0);
2921
2922 /* Our implementation of VK_KHR_multiview uses instancing to draw the
2923 * different views. We need to multiply instanceCount by the view count.
2924 */
2925 instanceCount *= anv_subpass_view_count(cmd_buffer->state.subpass);
2926
2927 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
2928 prim.PredicateEnable = cmd_buffer->state.conditional_render_enabled;
2929 prim.VertexAccessType = SEQUENTIAL;
2930 prim.PrimitiveTopologyType = pipeline->topology;
2931 prim.VertexCountPerInstance = vertexCount;
2932 prim.StartVertexLocation = firstVertex;
2933 prim.InstanceCount = instanceCount;
2934 prim.StartInstanceLocation = firstInstance;
2935 prim.BaseVertexLocation = 0;
2936 }
2937 }
2938
2939 void genX(CmdDrawIndexed)(
2940 VkCommandBuffer commandBuffer,
2941 uint32_t indexCount,
2942 uint32_t instanceCount,
2943 uint32_t firstIndex,
2944 int32_t vertexOffset,
2945 uint32_t firstInstance)
2946 {
2947 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
2948 struct anv_pipeline *pipeline = cmd_buffer->state.gfx.base.pipeline;
2949 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
2950
2951 if (anv_batch_has_error(&cmd_buffer->batch))
2952 return;
2953
2954 genX(cmd_buffer_flush_state)(cmd_buffer);
2955
2956 if (cmd_buffer->state.conditional_render_enabled)
2957 genX(cmd_emit_conditional_render_predicate)(cmd_buffer);
2958
2959 if (vs_prog_data->uses_firstvertex ||
2960 vs_prog_data->uses_baseinstance)
2961 emit_base_vertex_instance(cmd_buffer, vertexOffset, firstInstance);
2962 if (vs_prog_data->uses_drawid)
2963 emit_draw_index(cmd_buffer, 0);
2964
2965 /* Our implementation of VK_KHR_multiview uses instancing to draw the
2966 * different views. We need to multiply instanceCount by the view count.
2967 */
2968 instanceCount *= anv_subpass_view_count(cmd_buffer->state.subpass);
2969
2970 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
2971 prim.PredicateEnable = cmd_buffer->state.conditional_render_enabled;
2972 prim.VertexAccessType = RANDOM;
2973 prim.PrimitiveTopologyType = pipeline->topology;
2974 prim.VertexCountPerInstance = indexCount;
2975 prim.StartVertexLocation = firstIndex;
2976 prim.InstanceCount = instanceCount;
2977 prim.StartInstanceLocation = firstInstance;
2978 prim.BaseVertexLocation = vertexOffset;
2979 }
2980 }
2981
2982 /* Auto-Draw / Indirect Registers */
2983 #define GEN7_3DPRIM_END_OFFSET 0x2420
2984 #define GEN7_3DPRIM_START_VERTEX 0x2430
2985 #define GEN7_3DPRIM_VERTEX_COUNT 0x2434
2986 #define GEN7_3DPRIM_INSTANCE_COUNT 0x2438
2987 #define GEN7_3DPRIM_START_INSTANCE 0x243C
2988 #define GEN7_3DPRIM_BASE_VERTEX 0x2440
2989
2990 void genX(CmdDrawIndirectByteCountEXT)(
2991 VkCommandBuffer commandBuffer,
2992 uint32_t instanceCount,
2993 uint32_t firstInstance,
2994 VkBuffer counterBuffer,
2995 VkDeviceSize counterBufferOffset,
2996 uint32_t counterOffset,
2997 uint32_t vertexStride)
2998 {
2999 #if GEN_IS_HASWELL || GEN_GEN >= 8
3000 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
3001 ANV_FROM_HANDLE(anv_buffer, counter_buffer, counterBuffer);
3002 struct anv_pipeline *pipeline = cmd_buffer->state.gfx.base.pipeline;
3003 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
3004
3005 /* firstVertex is always zero for this draw function */
3006 const uint32_t firstVertex = 0;
3007
3008 if (anv_batch_has_error(&cmd_buffer->batch))
3009 return;
3010
3011 genX(cmd_buffer_flush_state)(cmd_buffer);
3012
3013 if (vs_prog_data->uses_firstvertex ||
3014 vs_prog_data->uses_baseinstance)
3015 emit_base_vertex_instance(cmd_buffer, firstVertex, firstInstance);
3016 if (vs_prog_data->uses_drawid)
3017 emit_draw_index(cmd_buffer, 0);
3018
3019 /* Our implementation of VK_KHR_multiview uses instancing to draw the
3020 * different views. We need to multiply instanceCount by the view count.
3021 */
3022 instanceCount *= anv_subpass_view_count(cmd_buffer->state.subpass);
3023
3024 struct gen_mi_builder b;
3025 gen_mi_builder_init(&b, &cmd_buffer->batch);
3026 struct gen_mi_value count =
3027 gen_mi_mem32(anv_address_add(counter_buffer->address,
3028 counterBufferOffset));
3029 if (counterOffset)
3030 count = gen_mi_isub(&b, count, gen_mi_imm(counterOffset));
3031 count = gen_mi_udiv32_imm(&b, count, vertexStride);
3032 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_VERTEX_COUNT), count);
3033
3034 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_START_VERTEX),
3035 gen_mi_imm(firstVertex));
3036 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_INSTANCE_COUNT),
3037 gen_mi_imm(instanceCount));
3038 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_START_INSTANCE),
3039 gen_mi_imm(firstInstance));
3040 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_BASE_VERTEX), gen_mi_imm(0));
3041
3042 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
3043 prim.IndirectParameterEnable = true;
3044 prim.VertexAccessType = SEQUENTIAL;
3045 prim.PrimitiveTopologyType = pipeline->topology;
3046 }
3047 #endif /* GEN_IS_HASWELL || GEN_GEN >= 8 */
3048 }
3049
3050 static void
3051 load_indirect_parameters(struct anv_cmd_buffer *cmd_buffer,
3052 struct anv_address addr,
3053 bool indexed)
3054 {
3055 struct gen_mi_builder b;
3056 gen_mi_builder_init(&b, &cmd_buffer->batch);
3057
3058 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_VERTEX_COUNT),
3059 gen_mi_mem32(anv_address_add(addr, 0)));
3060
3061 struct gen_mi_value instance_count = gen_mi_mem32(anv_address_add(addr, 4));
3062 unsigned view_count = anv_subpass_view_count(cmd_buffer->state.subpass);
3063 if (view_count > 1) {
3064 #if GEN_IS_HASWELL || GEN_GEN >= 8
3065 instance_count = gen_mi_imul_imm(&b, instance_count, view_count);
3066 #else
3067 anv_finishme("Multiview + indirect draw requires MI_MATH; "
3068 "MI_MATH is not supported on Ivy Bridge");
3069 #endif
3070 }
3071 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_INSTANCE_COUNT), instance_count);
3072
3073 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_START_VERTEX),
3074 gen_mi_mem32(anv_address_add(addr, 8)));
3075
3076 if (indexed) {
3077 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_BASE_VERTEX),
3078 gen_mi_mem32(anv_address_add(addr, 12)));
3079 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_START_INSTANCE),
3080 gen_mi_mem32(anv_address_add(addr, 16)));
3081 } else {
3082 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_START_INSTANCE),
3083 gen_mi_mem32(anv_address_add(addr, 12)));
3084 gen_mi_store(&b, gen_mi_reg32(GEN7_3DPRIM_BASE_VERTEX), gen_mi_imm(0));
3085 }
3086 }
3087
3088 void genX(CmdDrawIndirect)(
3089 VkCommandBuffer commandBuffer,
3090 VkBuffer _buffer,
3091 VkDeviceSize offset,
3092 uint32_t drawCount,
3093 uint32_t stride)
3094 {
3095 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
3096 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
3097 struct anv_pipeline *pipeline = cmd_buffer->state.gfx.base.pipeline;
3098 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
3099
3100 if (anv_batch_has_error(&cmd_buffer->batch))
3101 return;
3102
3103 genX(cmd_buffer_flush_state)(cmd_buffer);
3104
3105 if (cmd_buffer->state.conditional_render_enabled)
3106 genX(cmd_emit_conditional_render_predicate)(cmd_buffer);
3107
3108 for (uint32_t i = 0; i < drawCount; i++) {
3109 struct anv_address draw = anv_address_add(buffer->address, offset);
3110
3111 if (vs_prog_data->uses_firstvertex ||
3112 vs_prog_data->uses_baseinstance)
3113 emit_base_vertex_instance_bo(cmd_buffer, anv_address_add(draw, 8));
3114 if (vs_prog_data->uses_drawid)
3115 emit_draw_index(cmd_buffer, i);
3116
3117 load_indirect_parameters(cmd_buffer, draw, false);
3118
3119 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
3120 prim.IndirectParameterEnable = true;
3121 prim.PredicateEnable = cmd_buffer->state.conditional_render_enabled;
3122 prim.VertexAccessType = SEQUENTIAL;
3123 prim.PrimitiveTopologyType = pipeline->topology;
3124 }
3125
3126 offset += stride;
3127 }
3128 }
3129
3130 void genX(CmdDrawIndexedIndirect)(
3131 VkCommandBuffer commandBuffer,
3132 VkBuffer _buffer,
3133 VkDeviceSize offset,
3134 uint32_t drawCount,
3135 uint32_t stride)
3136 {
3137 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
3138 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
3139 struct anv_pipeline *pipeline = cmd_buffer->state.gfx.base.pipeline;
3140 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
3141
3142 if (anv_batch_has_error(&cmd_buffer->batch))
3143 return;
3144
3145 genX(cmd_buffer_flush_state)(cmd_buffer);
3146
3147 if (cmd_buffer->state.conditional_render_enabled)
3148 genX(cmd_emit_conditional_render_predicate)(cmd_buffer);
3149
3150 for (uint32_t i = 0; i < drawCount; i++) {
3151 struct anv_address draw = anv_address_add(buffer->address, offset);
3152
3153 /* TODO: We need to stomp base vertex to 0 somehow */
3154 if (vs_prog_data->uses_firstvertex ||
3155 vs_prog_data->uses_baseinstance)
3156 emit_base_vertex_instance_bo(cmd_buffer, anv_address_add(draw, 12));
3157 if (vs_prog_data->uses_drawid)
3158 emit_draw_index(cmd_buffer, i);
3159
3160 load_indirect_parameters(cmd_buffer, draw, true);
3161
3162 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
3163 prim.IndirectParameterEnable = true;
3164 prim.PredicateEnable = cmd_buffer->state.conditional_render_enabled;
3165 prim.VertexAccessType = RANDOM;
3166 prim.PrimitiveTopologyType = pipeline->topology;
3167 }
3168
3169 offset += stride;
3170 }
3171 }
3172
3173 #define TMP_DRAW_COUNT_REG 0x2670 /* MI_ALU_REG14 */
3174
3175 static void
3176 prepare_for_draw_count_predicate(struct anv_cmd_buffer *cmd_buffer,
3177 struct anv_address count_address,
3178 const bool conditional_render_enabled)
3179 {
3180 struct gen_mi_builder b;
3181 gen_mi_builder_init(&b, &cmd_buffer->batch);
3182
3183 if (conditional_render_enabled) {
3184 #if GEN_GEN >= 8 || GEN_IS_HASWELL
3185 gen_mi_store(&b, gen_mi_reg64(TMP_DRAW_COUNT_REG),
3186 gen_mi_mem32(count_address));
3187 #endif
3188 } else {
3189 /* Upload the current draw count from the draw parameters buffer to
3190 * MI_PREDICATE_SRC0.
3191 */
3192 gen_mi_store(&b, gen_mi_reg64(MI_PREDICATE_SRC0),
3193 gen_mi_mem32(count_address));
3194
3195 gen_mi_store(&b, gen_mi_reg32(MI_PREDICATE_SRC1 + 4), gen_mi_imm(0));
3196 }
3197 }
3198
3199 static void
3200 emit_draw_count_predicate(struct anv_cmd_buffer *cmd_buffer,
3201 uint32_t draw_index)
3202 {
3203 struct gen_mi_builder b;
3204 gen_mi_builder_init(&b, &cmd_buffer->batch);
3205
3206 /* Upload the index of the current primitive to MI_PREDICATE_SRC1. */
3207 gen_mi_store(&b, gen_mi_reg32(MI_PREDICATE_SRC1), gen_mi_imm(draw_index));
3208
3209 if (draw_index == 0) {
3210 anv_batch_emit(&cmd_buffer->batch, GENX(MI_PREDICATE), mip) {
3211 mip.LoadOperation = LOAD_LOADINV;
3212 mip.CombineOperation = COMBINE_SET;
3213 mip.CompareOperation = COMPARE_SRCS_EQUAL;
3214 }
3215 } else {
3216 /* While draw_index < draw_count the predicate's result will be
3217 * (draw_index == draw_count) ^ TRUE = TRUE
3218 * When draw_index == draw_count the result is
3219 * (TRUE) ^ TRUE = FALSE
3220 * After this all results will be:
3221 * (FALSE) ^ FALSE = FALSE
3222 */
3223 anv_batch_emit(&cmd_buffer->batch, GENX(MI_PREDICATE), mip) {
3224 mip.LoadOperation = LOAD_LOAD;
3225 mip.CombineOperation = COMBINE_XOR;
3226 mip.CompareOperation = COMPARE_SRCS_EQUAL;
3227 }
3228 }
3229 }
3230
3231 #if GEN_GEN >= 8 || GEN_IS_HASWELL
3232 static void
3233 emit_draw_count_predicate_with_conditional_render(
3234 struct anv_cmd_buffer *cmd_buffer,
3235 uint32_t draw_index)
3236 {
3237 struct gen_mi_builder b;
3238 gen_mi_builder_init(&b, &cmd_buffer->batch);
3239
3240 struct gen_mi_value pred = gen_mi_ult(&b, gen_mi_imm(draw_index),
3241 gen_mi_reg64(TMP_DRAW_COUNT_REG));
3242 pred = gen_mi_iand(&b, pred, gen_mi_reg64(ANV_PREDICATE_RESULT_REG));
3243
3244 #if GEN_GEN >= 8
3245 gen_mi_store(&b, gen_mi_reg64(MI_PREDICATE_RESULT), pred);
3246 #else
3247 /* MI_PREDICATE_RESULT is not whitelisted in i915 command parser
3248 * so we emit MI_PREDICATE to set it.
3249 */
3250
3251 gen_mi_store(&b, gen_mi_reg64(MI_PREDICATE_SRC0), pred);
3252 gen_mi_store(&b, gen_mi_reg64(MI_PREDICATE_SRC1), gen_mi_imm(0));
3253
3254 anv_batch_emit(&cmd_buffer->batch, GENX(MI_PREDICATE), mip) {
3255 mip.LoadOperation = LOAD_LOADINV;
3256 mip.CombineOperation = COMBINE_SET;
3257 mip.CompareOperation = COMPARE_SRCS_EQUAL;
3258 }
3259 #endif
3260 }
3261 #endif
3262
3263 void genX(CmdDrawIndirectCountKHR)(
3264 VkCommandBuffer commandBuffer,
3265 VkBuffer _buffer,
3266 VkDeviceSize offset,
3267 VkBuffer _countBuffer,
3268 VkDeviceSize countBufferOffset,
3269 uint32_t maxDrawCount,
3270 uint32_t stride)
3271 {
3272 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
3273 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
3274 ANV_FROM_HANDLE(anv_buffer, count_buffer, _countBuffer);
3275 struct anv_cmd_state *cmd_state = &cmd_buffer->state;
3276 struct anv_pipeline *pipeline = cmd_state->gfx.base.pipeline;
3277 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
3278
3279 if (anv_batch_has_error(&cmd_buffer->batch))
3280 return;
3281
3282 genX(cmd_buffer_flush_state)(cmd_buffer);
3283
3284 struct anv_address count_address =
3285 anv_address_add(count_buffer->address, countBufferOffset);
3286
3287 prepare_for_draw_count_predicate(cmd_buffer, count_address,
3288 cmd_state->conditional_render_enabled);
3289
3290 for (uint32_t i = 0; i < maxDrawCount; i++) {
3291 struct anv_address draw = anv_address_add(buffer->address, offset);
3292
3293 #if GEN_GEN >= 8 || GEN_IS_HASWELL
3294 if (cmd_state->conditional_render_enabled) {
3295 emit_draw_count_predicate_with_conditional_render(cmd_buffer, i);
3296 } else {
3297 emit_draw_count_predicate(cmd_buffer, i);
3298 }
3299 #else
3300 emit_draw_count_predicate(cmd_buffer, i);
3301 #endif
3302
3303 if (vs_prog_data->uses_firstvertex ||
3304 vs_prog_data->uses_baseinstance)
3305 emit_base_vertex_instance_bo(cmd_buffer, anv_address_add(draw, 8));
3306 if (vs_prog_data->uses_drawid)
3307 emit_draw_index(cmd_buffer, i);
3308
3309 load_indirect_parameters(cmd_buffer, draw, false);
3310
3311 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
3312 prim.IndirectParameterEnable = true;
3313 prim.PredicateEnable = true;
3314 prim.VertexAccessType = SEQUENTIAL;
3315 prim.PrimitiveTopologyType = pipeline->topology;
3316 }
3317
3318 offset += stride;
3319 }
3320 }
3321
3322 void genX(CmdDrawIndexedIndirectCountKHR)(
3323 VkCommandBuffer commandBuffer,
3324 VkBuffer _buffer,
3325 VkDeviceSize offset,
3326 VkBuffer _countBuffer,
3327 VkDeviceSize countBufferOffset,
3328 uint32_t maxDrawCount,
3329 uint32_t stride)
3330 {
3331 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
3332 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
3333 ANV_FROM_HANDLE(anv_buffer, count_buffer, _countBuffer);
3334 struct anv_cmd_state *cmd_state = &cmd_buffer->state;
3335 struct anv_pipeline *pipeline = cmd_state->gfx.base.pipeline;
3336 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
3337
3338 if (anv_batch_has_error(&cmd_buffer->batch))
3339 return;
3340
3341 genX(cmd_buffer_flush_state)(cmd_buffer);
3342
3343 struct anv_address count_address =
3344 anv_address_add(count_buffer->address, countBufferOffset);
3345
3346 prepare_for_draw_count_predicate(cmd_buffer, count_address,
3347 cmd_state->conditional_render_enabled);
3348
3349 for (uint32_t i = 0; i < maxDrawCount; i++) {
3350 struct anv_address draw = anv_address_add(buffer->address, offset);
3351
3352 #if GEN_GEN >= 8 || GEN_IS_HASWELL
3353 if (cmd_state->conditional_render_enabled) {
3354 emit_draw_count_predicate_with_conditional_render(cmd_buffer, i);
3355 } else {
3356 emit_draw_count_predicate(cmd_buffer, i);
3357 }
3358 #else
3359 emit_draw_count_predicate(cmd_buffer, i);
3360 #endif
3361
3362 /* TODO: We need to stomp base vertex to 0 somehow */
3363 if (vs_prog_data->uses_firstvertex ||
3364 vs_prog_data->uses_baseinstance)
3365 emit_base_vertex_instance_bo(cmd_buffer, anv_address_add(draw, 12));
3366 if (vs_prog_data->uses_drawid)
3367 emit_draw_index(cmd_buffer, i);
3368
3369 load_indirect_parameters(cmd_buffer, draw, true);
3370
3371 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
3372 prim.IndirectParameterEnable = true;
3373 prim.PredicateEnable = true;
3374 prim.VertexAccessType = RANDOM;
3375 prim.PrimitiveTopologyType = pipeline->topology;
3376 }
3377
3378 offset += stride;
3379 }
3380 }
3381
3382 void genX(CmdBeginTransformFeedbackEXT)(
3383 VkCommandBuffer commandBuffer,
3384 uint32_t firstCounterBuffer,
3385 uint32_t counterBufferCount,
3386 const VkBuffer* pCounterBuffers,
3387 const VkDeviceSize* pCounterBufferOffsets)
3388 {
3389 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
3390
3391 assert(firstCounterBuffer < MAX_XFB_BUFFERS);
3392 assert(counterBufferCount <= MAX_XFB_BUFFERS);
3393 assert(firstCounterBuffer + counterBufferCount <= MAX_XFB_BUFFERS);
3394
3395 /* From the SKL PRM Vol. 2c, SO_WRITE_OFFSET:
3396 *
3397 * "Ssoftware must ensure that no HW stream output operations can be in
3398 * process or otherwise pending at the point that the MI_LOAD/STORE
3399 * commands are processed. This will likely require a pipeline flush."
3400 */
3401 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_CS_STALL_BIT;
3402 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
3403
3404 for (uint32_t idx = 0; idx < MAX_XFB_BUFFERS; idx++) {
3405 /* If we have a counter buffer, this is a resume so we need to load the
3406 * value into the streamout offset register. Otherwise, this is a begin
3407 * and we need to reset it to zero.
3408 */
3409 if (pCounterBuffers &&
3410 idx >= firstCounterBuffer &&
3411 idx - firstCounterBuffer < counterBufferCount &&
3412 pCounterBuffers[idx - firstCounterBuffer] != VK_NULL_HANDLE) {
3413 uint32_t cb_idx = idx - firstCounterBuffer;
3414 ANV_FROM_HANDLE(anv_buffer, counter_buffer, pCounterBuffers[cb_idx]);
3415 uint64_t offset = pCounterBufferOffsets ?
3416 pCounterBufferOffsets[cb_idx] : 0;
3417
3418 anv_batch_emit(&cmd_buffer->batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
3419 lrm.RegisterAddress = GENX(SO_WRITE_OFFSET0_num) + idx * 4;
3420 lrm.MemoryAddress = anv_address_add(counter_buffer->address,
3421 offset);
3422 }
3423 } else {
3424 anv_batch_emit(&cmd_buffer->batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
3425 lri.RegisterOffset = GENX(SO_WRITE_OFFSET0_num) + idx * 4;
3426 lri.DataDWord = 0;
3427 }
3428 }
3429 }
3430
3431 cmd_buffer->state.xfb_enabled = true;
3432 cmd_buffer->state.gfx.dirty |= ANV_CMD_DIRTY_XFB_ENABLE;
3433 }
3434
3435 void genX(CmdEndTransformFeedbackEXT)(
3436 VkCommandBuffer commandBuffer,
3437 uint32_t firstCounterBuffer,
3438 uint32_t counterBufferCount,
3439 const VkBuffer* pCounterBuffers,
3440 const VkDeviceSize* pCounterBufferOffsets)
3441 {
3442 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
3443
3444 assert(firstCounterBuffer < MAX_XFB_BUFFERS);
3445 assert(counterBufferCount <= MAX_XFB_BUFFERS);
3446 assert(firstCounterBuffer + counterBufferCount <= MAX_XFB_BUFFERS);
3447
3448 /* From the SKL PRM Vol. 2c, SO_WRITE_OFFSET:
3449 *
3450 * "Ssoftware must ensure that no HW stream output operations can be in
3451 * process or otherwise pending at the point that the MI_LOAD/STORE
3452 * commands are processed. This will likely require a pipeline flush."
3453 */
3454 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_CS_STALL_BIT;
3455 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
3456
3457 for (uint32_t cb_idx = 0; cb_idx < counterBufferCount; cb_idx++) {
3458 unsigned idx = firstCounterBuffer + cb_idx;
3459
3460 /* If we have a counter buffer, this is a resume so we need to load the
3461 * value into the streamout offset register. Otherwise, this is a begin
3462 * and we need to reset it to zero.
3463 */
3464 if (pCounterBuffers &&
3465 cb_idx < counterBufferCount &&
3466 pCounterBuffers[cb_idx] != VK_NULL_HANDLE) {
3467 ANV_FROM_HANDLE(anv_buffer, counter_buffer, pCounterBuffers[cb_idx]);
3468 uint64_t offset = pCounterBufferOffsets ?
3469 pCounterBufferOffsets[cb_idx] : 0;
3470
3471 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), srm) {
3472 srm.MemoryAddress = anv_address_add(counter_buffer->address,
3473 offset);
3474 srm.RegisterAddress = GENX(SO_WRITE_OFFSET0_num) + idx * 4;
3475 }
3476 }
3477 }
3478
3479 cmd_buffer->state.xfb_enabled = false;
3480 cmd_buffer->state.gfx.dirty |= ANV_CMD_DIRTY_XFB_ENABLE;
3481 }
3482
3483 static VkResult
3484 flush_compute_descriptor_set(struct anv_cmd_buffer *cmd_buffer)
3485 {
3486 struct anv_pipeline *pipeline = cmd_buffer->state.compute.base.pipeline;
3487 struct anv_state surfaces = { 0, }, samplers = { 0, };
3488 VkResult result;
3489
3490 result = emit_binding_table(cmd_buffer, MESA_SHADER_COMPUTE, &surfaces);
3491 if (result != VK_SUCCESS) {
3492 assert(result == VK_ERROR_OUT_OF_DEVICE_MEMORY);
3493
3494 result = anv_cmd_buffer_new_binding_table_block(cmd_buffer);
3495 if (result != VK_SUCCESS)
3496 return result;
3497
3498 /* Re-emit state base addresses so we get the new surface state base
3499 * address before we start emitting binding tables etc.
3500 */
3501 genX(cmd_buffer_emit_state_base_address)(cmd_buffer);
3502
3503 result = emit_binding_table(cmd_buffer, MESA_SHADER_COMPUTE, &surfaces);
3504 if (result != VK_SUCCESS) {
3505 anv_batch_set_error(&cmd_buffer->batch, result);
3506 return result;
3507 }
3508 }
3509
3510 result = emit_samplers(cmd_buffer, MESA_SHADER_COMPUTE, &samplers);
3511 if (result != VK_SUCCESS) {
3512 anv_batch_set_error(&cmd_buffer->batch, result);
3513 return result;
3514 }
3515
3516 uint32_t iface_desc_data_dw[GENX(INTERFACE_DESCRIPTOR_DATA_length)];
3517 struct GENX(INTERFACE_DESCRIPTOR_DATA) desc = {
3518 .BindingTablePointer = surfaces.offset,
3519 .SamplerStatePointer = samplers.offset,
3520 };
3521 GENX(INTERFACE_DESCRIPTOR_DATA_pack)(NULL, iface_desc_data_dw, &desc);
3522
3523 struct anv_state state =
3524 anv_cmd_buffer_merge_dynamic(cmd_buffer, iface_desc_data_dw,
3525 pipeline->interface_descriptor_data,
3526 GENX(INTERFACE_DESCRIPTOR_DATA_length),
3527 64);
3528
3529 uint32_t size = GENX(INTERFACE_DESCRIPTOR_DATA_length) * sizeof(uint32_t);
3530 anv_batch_emit(&cmd_buffer->batch,
3531 GENX(MEDIA_INTERFACE_DESCRIPTOR_LOAD), mid) {
3532 mid.InterfaceDescriptorTotalLength = size;
3533 mid.InterfaceDescriptorDataStartAddress = state.offset;
3534 }
3535
3536 return VK_SUCCESS;
3537 }
3538
3539 void
3540 genX(cmd_buffer_flush_compute_state)(struct anv_cmd_buffer *cmd_buffer)
3541 {
3542 struct anv_pipeline *pipeline = cmd_buffer->state.compute.base.pipeline;
3543 VkResult result;
3544
3545 assert(pipeline->active_stages == VK_SHADER_STAGE_COMPUTE_BIT);
3546
3547 genX(cmd_buffer_config_l3)(cmd_buffer, pipeline->urb.l3_config);
3548
3549 genX(flush_pipeline_select_gpgpu)(cmd_buffer);
3550
3551 if (cmd_buffer->state.compute.pipeline_dirty) {
3552 /* From the Sky Lake PRM Vol 2a, MEDIA_VFE_STATE:
3553 *
3554 * "A stalling PIPE_CONTROL is required before MEDIA_VFE_STATE unless
3555 * the only bits that are changed are scoreboard related: Scoreboard
3556 * Enable, Scoreboard Type, Scoreboard Mask, Scoreboard * Delta. For
3557 * these scoreboard related states, a MEDIA_STATE_FLUSH is
3558 * sufficient."
3559 */
3560 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_CS_STALL_BIT;
3561 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
3562
3563 anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
3564 }
3565
3566 if ((cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_COMPUTE_BIT) ||
3567 cmd_buffer->state.compute.pipeline_dirty) {
3568 /* FIXME: figure out descriptors for gen7 */
3569 result = flush_compute_descriptor_set(cmd_buffer);
3570 if (result != VK_SUCCESS)
3571 return;
3572
3573 cmd_buffer->state.descriptors_dirty &= ~VK_SHADER_STAGE_COMPUTE_BIT;
3574 }
3575
3576 if (cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_COMPUTE_BIT) {
3577 struct anv_state push_state =
3578 anv_cmd_buffer_cs_push_constants(cmd_buffer);
3579
3580 if (push_state.alloc_size) {
3581 anv_batch_emit(&cmd_buffer->batch, GENX(MEDIA_CURBE_LOAD), curbe) {
3582 curbe.CURBETotalDataLength = push_state.alloc_size;
3583 curbe.CURBEDataStartAddress = push_state.offset;
3584 }
3585 }
3586
3587 cmd_buffer->state.push_constants_dirty &= ~VK_SHADER_STAGE_COMPUTE_BIT;
3588 }
3589
3590 cmd_buffer->state.compute.pipeline_dirty = false;
3591
3592 genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
3593 }
3594
3595 #if GEN_GEN == 7
3596
3597 static VkResult
3598 verify_cmd_parser(const struct anv_device *device,
3599 int required_version,
3600 const char *function)
3601 {
3602 if (device->instance->physicalDevice.cmd_parser_version < required_version) {
3603 return vk_errorf(device->instance, device->instance,
3604 VK_ERROR_FEATURE_NOT_PRESENT,
3605 "cmd parser version %d is required for %s",
3606 required_version, function);
3607 } else {
3608 return VK_SUCCESS;
3609 }
3610 }
3611