projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv/cmd_buffer: Pull the core of flush_state into genX_cmd_buffer
[mesa.git] / src / intel / vulkan / genX_cmd_buffer.c
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26
27 #include "anv_private.h"
28
29 #include "genxml/gen_macros.h"
30 #include "genxml/genX_pack.h"
31
32 void
33 genX(cmd_buffer_emit_state_base_address)(struct anv_cmd_buffer *cmd_buffer)
34 {
35 struct anv_device *device = cmd_buffer->device;
36 struct anv_bo *scratch_bo = NULL;
37
38 cmd_buffer->state.scratch_size =
39 anv_block_pool_size(&device->scratch_block_pool);
40 if (cmd_buffer->state.scratch_size > 0)
41 scratch_bo = &device->scratch_block_pool.bo;
42
43 /* XXX: Do we need this on more than just BDW? */
44 #if (GEN_GEN >= 8)
45 /* Emit a render target cache flush.
46 *
47 * This isn't documented anywhere in the PRM. However, it seems to be
48 * necessary prior to changing the surface state base adress. Without
49 * this, we get GPU hangs when using multi-level command buffers which
50 * clear depth, reset state base address, and then go render stuff.
51 */
52 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
53 .RenderTargetCacheFlushEnable = true);
54 #endif
55
56 anv_batch_emit(&cmd_buffer->batch, GENX(STATE_BASE_ADDRESS),
57 .GeneralStateBaseAddress = { scratch_bo, 0 },
58 .GeneralStateMemoryObjectControlState = GENX(MOCS),
59 .GeneralStateBaseAddressModifyEnable = true,
60
61 .SurfaceStateBaseAddress = anv_cmd_buffer_surface_base_address(cmd_buffer),
62 .SurfaceStateMemoryObjectControlState = GENX(MOCS),
63 .SurfaceStateBaseAddressModifyEnable = true,
64
65 .DynamicStateBaseAddress = { &device->dynamic_state_block_pool.bo, 0 },
66 .DynamicStateMemoryObjectControlState = GENX(MOCS),
67 .DynamicStateBaseAddressModifyEnable = true,
68
69 .IndirectObjectBaseAddress = { NULL, 0 },
70 .IndirectObjectMemoryObjectControlState = GENX(MOCS),
71 .IndirectObjectBaseAddressModifyEnable = true,
72
73 .InstructionBaseAddress = { &device->instruction_block_pool.bo, 0 },
74 .InstructionMemoryObjectControlState = GENX(MOCS),
75 .InstructionBaseAddressModifyEnable = true,
76
77 # if (GEN_GEN >= 8)
78 /* Broadwell requires that we specify a buffer size for a bunch of
79 * these fields. However, since we will be growing the BO's live, we
80 * just set them all to the maximum.
81 */
82 .GeneralStateBufferSize = 0xfffff,
83 .GeneralStateBufferSizeModifyEnable = true,
84 .DynamicStateBufferSize = 0xfffff,
85 .DynamicStateBufferSizeModifyEnable = true,
86 .IndirectObjectBufferSize = 0xfffff,
87 .IndirectObjectBufferSizeModifyEnable = true,
88 .InstructionBufferSize = 0xfffff,
89 .InstructionBuffersizeModifyEnable = true,
90 # endif
91 );
92
93 /* After re-setting the surface state base address, we have to do some
94 * cache flusing so that the sampler engine will pick up the new
95 * SURFACE_STATE objects and binding tables. From the Broadwell PRM,
96 * Shared Function > 3D Sampler > State > State Caching (page 96):
97 *
98 * Coherency with system memory in the state cache, like the texture
99 * cache is handled partially by software. It is expected that the
100 * command stream or shader will issue Cache Flush operation or
101 * Cache_Flush sampler message to ensure that the L1 cache remains
102 * coherent with system memory.
103 *
104 * [...]
105 *
106 * Whenever the value of the Dynamic_State_Base_Addr,
107 * Surface_State_Base_Addr are altered, the L1 state cache must be
108 * invalidated to ensure the new surface or sampler state is fetched
109 * from system memory.
110 *
111 * The PIPE_CONTROL command has a "State Cache Invalidation Enable" bit
112 * which, according the PIPE_CONTROL instruction documentation in the
113 * Broadwell PRM:
114 *
115 * Setting this bit is independent of any other bit in this packet.
116 * This bit controls the invalidation of the L1 and L2 state caches
117 * at the top of the pipe i.e. at the parsing time.
118 *
119 * Unfortunately, experimentation seems to indicate that state cache
120 * invalidation through a PIPE_CONTROL does nothing whatsoever in
121 * regards to surface state and binding tables. In stead, it seems that
122 * invalidating the texture cache is what is actually needed.
123 *
124 * XXX: As far as we have been able to determine through
125 * experimentation, shows that flush the texture cache appears to be
126 * sufficient. The theory here is that all of the sampling/rendering
127 * units cache the binding table in the texture cache. However, we have
128 * yet to be able to actually confirm this.
129 */
130 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
131 .TextureCacheInvalidationEnable = true);
132 }
133
134 void genX(CmdPipelineBarrier)(
135 VkCommandBuffer commandBuffer,
136 VkPipelineStageFlags srcStageMask,
137 VkPipelineStageFlags destStageMask,
138 VkBool32 byRegion,
139 uint32_t memoryBarrierCount,
140 const VkMemoryBarrier* pMemoryBarriers,
141 uint32_t bufferMemoryBarrierCount,
142 const VkBufferMemoryBarrier* pBufferMemoryBarriers,
143 uint32_t imageMemoryBarrierCount,
144 const VkImageMemoryBarrier* pImageMemoryBarriers)
145 {
146 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
147 uint32_t b, *dw;
148
149 /* XXX: Right now, we're really dumb and just flush whatever categories
150 * the app asks for. One of these days we may make this a bit better
151 * but right now that's all the hardware allows for in most areas.
152 */
153 VkAccessFlags src_flags = 0;
154 VkAccessFlags dst_flags = 0;
155
156 for (uint32_t i = 0; i < memoryBarrierCount; i++) {
157 src_flags |= pMemoryBarriers[i].srcAccessMask;
158 dst_flags |= pMemoryBarriers[i].dstAccessMask;
159 }
160
161 for (uint32_t i = 0; i < bufferMemoryBarrierCount; i++) {
162 src_flags |= pBufferMemoryBarriers[i].srcAccessMask;
163 dst_flags |= pBufferMemoryBarriers[i].dstAccessMask;
164 }
165
166 for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) {
167 src_flags |= pImageMemoryBarriers[i].srcAccessMask;
168 dst_flags |= pImageMemoryBarriers[i].dstAccessMask;
169 }
170
171 /* Mask out the Source access flags we care about */
172 const uint32_t src_mask =
173 VK_ACCESS_SHADER_WRITE_BIT |
174 VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT |
175 VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT |
176 VK_ACCESS_TRANSFER_WRITE_BIT;
177
178 src_flags = src_flags & src_mask;
179
180 /* Mask out the destination access flags we care about */
181 const uint32_t dst_mask =
182 VK_ACCESS_INDIRECT_COMMAND_READ_BIT |
183 VK_ACCESS_INDEX_READ_BIT |
184 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT |
185 VK_ACCESS_UNIFORM_READ_BIT |
186 VK_ACCESS_SHADER_READ_BIT |
187 VK_ACCESS_COLOR_ATTACHMENT_READ_BIT |
188 VK_ACCESS_TRANSFER_READ_BIT;
189
190 dst_flags = dst_flags & dst_mask;
191
192 /* The src flags represent how things were used previously. This is
193 * what we use for doing flushes.
194 */
195 struct GENX(PIPE_CONTROL) flush_cmd = {
196 GENX(PIPE_CONTROL_header),
197 .PostSyncOperation = NoWrite,
198 };
199
200 for_each_bit(b, src_flags) {
201 switch ((VkAccessFlagBits)(1 << b)) {
202 case VK_ACCESS_SHADER_WRITE_BIT:
203 flush_cmd.DCFlushEnable = true;
204 break;
205 case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT:
206 flush_cmd.RenderTargetCacheFlushEnable = true;
207 break;
208 case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT:
209 flush_cmd.DepthCacheFlushEnable = true;
210 break;
211 case VK_ACCESS_TRANSFER_WRITE_BIT:
212 flush_cmd.RenderTargetCacheFlushEnable = true;
213 flush_cmd.DepthCacheFlushEnable = true;
214 break;
215 default:
216 unreachable("should've masked this out by now");
217 }
218 }
219
220 /* If we end up doing two PIPE_CONTROLs, the first, flusing one also has to
221 * stall and wait for the flushing to finish, so we don't re-dirty the
222 * caches with in-flight rendering after the second PIPE_CONTROL
223 * invalidates.
224 */
225
226 if (dst_flags)
227 flush_cmd.CommandStreamerStallEnable = true;
228
229 if (src_flags && dst_flags) {
230 dw = anv_batch_emit_dwords(&cmd_buffer->batch, GENX(PIPE_CONTROL_length));
231 GENX(PIPE_CONTROL_pack)(&cmd_buffer->batch, dw, &flush_cmd);
232 }
233
234 /* The dst flags represent how things will be used in the future. This
235 * is what we use for doing cache invalidations.
236 */
237 struct GENX(PIPE_CONTROL) invalidate_cmd = {
238 GENX(PIPE_CONTROL_header),
239 .PostSyncOperation = NoWrite,
240 };
241
242 for_each_bit(b, dst_flags) {
243 switch ((VkAccessFlagBits)(1 << b)) {
244 case VK_ACCESS_INDIRECT_COMMAND_READ_BIT:
245 case VK_ACCESS_INDEX_READ_BIT:
246 case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT:
247 invalidate_cmd.VFCacheInvalidationEnable = true;
248 break;
249 case VK_ACCESS_UNIFORM_READ_BIT:
250 invalidate_cmd.ConstantCacheInvalidationEnable = true;
251 /* fallthrough */
252 case VK_ACCESS_SHADER_READ_BIT:
253 invalidate_cmd.TextureCacheInvalidationEnable = true;
254 break;
255 case VK_ACCESS_COLOR_ATTACHMENT_READ_BIT:
256 invalidate_cmd.TextureCacheInvalidationEnable = true;
257 break;
258 case VK_ACCESS_TRANSFER_READ_BIT:
259 invalidate_cmd.TextureCacheInvalidationEnable = true;
260 break;
261 default:
262 unreachable("should've masked this out by now");
263 }
264 }
265
266 if (dst_flags) {
267 dw = anv_batch_emit_dwords(&cmd_buffer->batch, GENX(PIPE_CONTROL_length));
268 GENX(PIPE_CONTROL_pack)(&cmd_buffer->batch, dw, &invalidate_cmd);
269 }
270 }
271
272 static uint32_t
273 cmd_buffer_flush_push_constants(struct anv_cmd_buffer *cmd_buffer)
274 {
275 static const uint32_t push_constant_opcodes[] = {
276 [MESA_SHADER_VERTEX] = 21,
277 [MESA_SHADER_TESS_CTRL] = 25, /* HS */
278 [MESA_SHADER_TESS_EVAL] = 26, /* DS */
279 [MESA_SHADER_GEOMETRY] = 22,
280 [MESA_SHADER_FRAGMENT] = 23,
281 [MESA_SHADER_COMPUTE] = 0,
282 };
283
284 VkShaderStageFlags flushed = 0;
285
286 anv_foreach_stage(stage, cmd_buffer->state.push_constants_dirty) {
287 if (stage == MESA_SHADER_COMPUTE)
288 continue;
289
290 struct anv_state state = anv_cmd_buffer_push_constants(cmd_buffer, stage);
291
292 if (state.offset == 0) {
293 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CONSTANT_VS),
294 ._3DCommandSubOpcode = push_constant_opcodes[stage]);
295 } else {
296 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CONSTANT_VS),
297 ._3DCommandSubOpcode = push_constant_opcodes[stage],
298 .ConstantBody = {
299 #if GEN_GEN >= 9
300 .PointerToConstantBuffer2 = { &cmd_buffer->device->dynamic_state_block_pool.bo, state.offset },
301 .ConstantBuffer2ReadLength = DIV_ROUND_UP(state.alloc_size, 32),
302 #else
303 .PointerToConstantBuffer0 = { .offset = state.offset },
304 .ConstantBuffer0ReadLength = DIV_ROUND_UP(state.alloc_size, 32),
305 #endif
306 });
307 }
308
309 flushed |= mesa_to_vk_shader_stage(stage);
310 }
311
312 cmd_buffer->state.push_constants_dirty &= ~VK_SHADER_STAGE_ALL_GRAPHICS;
313
314 return flushed;
315 }
316
317 void
318 genX(cmd_buffer_flush_state)(struct anv_cmd_buffer *cmd_buffer)
319 {
320 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
321 uint32_t *p;
322
323 uint32_t vb_emit = cmd_buffer->state.vb_dirty & pipeline->vb_used;
324
325 assert((pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT) == 0);
326
327 genX(cmd_buffer_config_l3)(cmd_buffer, false);
328
329 genX(flush_pipeline_select_3d)(cmd_buffer);
330
331 if (vb_emit) {
332 const uint32_t num_buffers = __builtin_popcount(vb_emit);
333 const uint32_t num_dwords = 1 + num_buffers * 4;
334
335 p = anv_batch_emitn(&cmd_buffer->batch, num_dwords,
336 GENX(3DSTATE_VERTEX_BUFFERS));
337 uint32_t vb, i = 0;
338 for_each_bit(vb, vb_emit) {
339 struct anv_buffer *buffer = cmd_buffer->state.vertex_bindings[vb].buffer;
340 uint32_t offset = cmd_buffer->state.vertex_bindings[vb].offset;
341
342 struct GENX(VERTEX_BUFFER_STATE) state = {
343 .VertexBufferIndex = vb,
344
345 #if GEN_GEN >= 8
346 .MemoryObjectControlState = GENX(MOCS),
347 #else
348 .BufferAccessType = pipeline->instancing_enable[vb] ? INSTANCEDATA : VERTEXDATA,
349 .InstanceDataStepRate = 1,
350 .VertexBufferMemoryObjectControlState = GENX(MOCS),
351 #endif
352
353 .AddressModifyEnable = true,
354 .BufferPitch = pipeline->binding_stride[vb],
355 .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
356
357 #if GEN_GEN >= 8
358 .BufferSize = buffer->size - offset
359 #else
360 .EndAddress = { buffer->bo, buffer->offset + buffer->size - 1},
361 #endif
362 };
363
364 GENX(VERTEX_BUFFER_STATE_pack)(&cmd_buffer->batch, &p[1 + i * 4], &state);
365 i++;
366 }
367 }
368
369 cmd_buffer->state.vb_dirty &= ~vb_emit;
370
371 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_PIPELINE) {
372 /* If somebody compiled a pipeline after starting a command buffer the
373 * scratch bo may have grown since we started this cmd buffer (and
374 * emitted STATE_BASE_ADDRESS). If we're binding that pipeline now,
375 * reemit STATE_BASE_ADDRESS so that we use the bigger scratch bo. */
376 if (cmd_buffer->state.scratch_size < pipeline->total_scratch)
377 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
378
379 anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
380
381 /* From the BDW PRM for 3DSTATE_PUSH_CONSTANT_ALLOC_VS:
382 *
383 * "The 3DSTATE_CONSTANT_VS must be reprogrammed prior to
384 * the next 3DPRIMITIVE command after programming the
385 * 3DSTATE_PUSH_CONSTANT_ALLOC_VS"
386 *
387 * Since 3DSTATE_PUSH_CONSTANT_ALLOC_VS is programmed as part of
388 * pipeline setup, we need to dirty push constants.
389 */
390 cmd_buffer->state.push_constants_dirty |= VK_SHADER_STAGE_ALL_GRAPHICS;
391 }
392
393 #if GEN_GEN <= 7
394 if (cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_VERTEX_BIT ||
395 cmd_buffer->state.push_constants_dirty & VK_SHADER_STAGE_VERTEX_BIT) {
396 /* From the IVB PRM Vol. 2, Part 1, Section 3.2.1:
397 *
398 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
399 * stall needs to be sent just prior to any 3DSTATE_VS,
400 * 3DSTATE_URB_VS, 3DSTATE_CONSTANT_VS,
401 * 3DSTATE_BINDING_TABLE_POINTER_VS,
402 * 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one
403 * PIPE_CONTROL needs to be sent before any combination of VS
404 * associated 3DSTATE."
405 */
406 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
407 .DepthStallEnable = true,
408 .PostSyncOperation = WriteImmediateData,
409 .Address = { &cmd_buffer->device->workaround_bo, 0 });
410 }
411 #endif
412
413 /* We emit the binding tables and sampler tables first, then emit push
414 * constants and then finally emit binding table and sampler table
415 * pointers. It has to happen in this order, since emitting the binding
416 * tables may change the push constants (in case of storage images). After
417 * emitting push constants, on SKL+ we have to emit the corresponding
418 * 3DSTATE_BINDING_TABLE_POINTER_* for the push constants to take effect.
419 */
420 uint32_t dirty = 0;
421 if (cmd_buffer->state.descriptors_dirty)
422 dirty = gen7_cmd_buffer_flush_descriptor_sets(cmd_buffer);
423
424 if (cmd_buffer->state.push_constants_dirty) {
425 #if GEN_GEN >= 9
426 /* On Sky Lake and later, the binding table pointers commands are
427 * what actually flush the changes to push constant state so we need
428 * to dirty them so they get re-emitted below.
429 */
430 dirty |= cmd_buffer_flush_push_constants(cmd_buffer);
431 #else
432 cmd_buffer_flush_push_constants(cmd_buffer);
433 #endif
434 }
435
436 if (dirty)
437 gen7_cmd_buffer_emit_descriptor_pointers(cmd_buffer, dirty);
438
439 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_VIEWPORT)
440 gen8_cmd_buffer_emit_viewport(cmd_buffer);
441
442 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_SCISSOR)
443 gen7_cmd_buffer_emit_scissor(cmd_buffer);
444
445 genX(cmd_buffer_flush_dynamic_state)(cmd_buffer);
446 }
447
448 static void
449 emit_base_vertex_instance_bo(struct anv_cmd_buffer *cmd_buffer,
450 struct anv_bo *bo, uint32_t offset)
451 {
452 uint32_t *p = anv_batch_emitn(&cmd_buffer->batch, 5,
453 GENX(3DSTATE_VERTEX_BUFFERS));
454
455 GENX(VERTEX_BUFFER_STATE_pack)(&cmd_buffer->batch, p + 1,
456 &(struct GENX(VERTEX_BUFFER_STATE)) {
457 .VertexBufferIndex = 32, /* Reserved for this */
458 .AddressModifyEnable = true,
459 .BufferPitch = 0,
460 #if (GEN_GEN >= 8)
461 .MemoryObjectControlState = GENX(MOCS),
462 .BufferStartingAddress = { bo, offset },
463 .BufferSize = 8
464 #else
465 .VertexBufferMemoryObjectControlState = GENX(MOCS),
466 .BufferStartingAddress = { bo, offset },
467 .EndAddress = { bo, offset + 8 },
468 #endif
469 });
470 }
471
472 static void
473 emit_base_vertex_instance(struct anv_cmd_buffer *cmd_buffer,
474 uint32_t base_vertex, uint32_t base_instance)
475 {
476 struct anv_state id_state =
477 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, 8, 4);
478
479 ((uint32_t *)id_state.map)[0] = base_vertex;
480 ((uint32_t *)id_state.map)[1] = base_instance;
481
482 if (!cmd_buffer->device->info.has_llc)
483 anv_state_clflush(id_state);
484
485 emit_base_vertex_instance_bo(cmd_buffer,
486 &cmd_buffer->device->dynamic_state_block_pool.bo, id_state.offset);
487 }
488
489 void genX(CmdDraw)(
490 VkCommandBuffer commandBuffer,
491 uint32_t vertexCount,
492 uint32_t instanceCount,
493 uint32_t firstVertex,
494 uint32_t firstInstance)
495 {
496 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
497 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
498 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
499
500 genX(cmd_buffer_flush_state)(cmd_buffer);
501
502 if (vs_prog_data->uses_basevertex || vs_prog_data->uses_baseinstance)
503 emit_base_vertex_instance(cmd_buffer, firstVertex, firstInstance);
504
505 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
506 .VertexAccessType = SEQUENTIAL,
507 .PrimitiveTopologyType = pipeline->topology,
508 .VertexCountPerInstance = vertexCount,
509 .StartVertexLocation = firstVertex,
510 .InstanceCount = instanceCount,
511 .StartInstanceLocation = firstInstance,
512 .BaseVertexLocation = 0);
513 }
514
515 void genX(CmdDrawIndexed)(
516 VkCommandBuffer commandBuffer,
517 uint32_t indexCount,
518 uint32_t instanceCount,
519 uint32_t firstIndex,
520 int32_t vertexOffset,
521 uint32_t firstInstance)
522 {
523 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
524 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
525 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
526
527 genX(cmd_buffer_flush_state)(cmd_buffer);
528
529 if (vs_prog_data->uses_basevertex || vs_prog_data->uses_baseinstance)
530 emit_base_vertex_instance(cmd_buffer, vertexOffset, firstInstance);
531
532 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
533 .VertexAccessType = RANDOM,
534 .PrimitiveTopologyType = pipeline->topology,
535 .VertexCountPerInstance = indexCount,
536 .StartVertexLocation = firstIndex,
537 .InstanceCount = instanceCount,
538 .StartInstanceLocation = firstInstance,
539 .BaseVertexLocation = vertexOffset);
540 }
541
542 /* Auto-Draw / Indirect Registers */
543 #define GEN7_3DPRIM_END_OFFSET 0x2420
544 #define GEN7_3DPRIM_START_VERTEX 0x2430
545 #define GEN7_3DPRIM_VERTEX_COUNT 0x2434
546 #define GEN7_3DPRIM_INSTANCE_COUNT 0x2438
547 #define GEN7_3DPRIM_START_INSTANCE 0x243C
548 #define GEN7_3DPRIM_BASE_VERTEX 0x2440
549
550 static void
551 emit_lrm(struct anv_batch *batch,
552 uint32_t reg, struct anv_bo *bo, uint32_t offset)
553 {
554 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM),
555 .RegisterAddress = reg,
556 .MemoryAddress = { bo, offset });
557 }
558
559 static void
560 emit_lri(struct anv_batch *batch, uint32_t reg, uint32_t imm)
561 {
562 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM),
563 .RegisterOffset = reg,
564 .DataDWord = imm);
565 }
566
567 void genX(CmdDrawIndirect)(
568 VkCommandBuffer commandBuffer,
569 VkBuffer _buffer,
570 VkDeviceSize offset,
571 uint32_t drawCount,
572 uint32_t stride)
573 {
574 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
575 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
576 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
577 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
578 struct anv_bo *bo = buffer->bo;
579 uint32_t bo_offset = buffer->offset + offset;
580
581 genX(cmd_buffer_flush_state)(cmd_buffer);
582
583 if (vs_prog_data->uses_basevertex || vs_prog_data->uses_baseinstance)
584 emit_base_vertex_instance_bo(cmd_buffer, bo, bo_offset + 8);
585
586 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
587 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
588 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
589 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 12);
590 emit_lri(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, 0);
591
592 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
593 .IndirectParameterEnable = true,
594 .VertexAccessType = SEQUENTIAL,
595 .PrimitiveTopologyType = pipeline->topology);
596 }
597
598 void genX(CmdDrawIndexedIndirect)(
599 VkCommandBuffer commandBuffer,
600 VkBuffer _buffer,
601 VkDeviceSize offset,
602 uint32_t drawCount,
603 uint32_t stride)
604 {
605 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
606 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
607 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
608 const struct brw_vs_prog_data *vs_prog_data = get_vs_prog_data(pipeline);
609 struct anv_bo *bo = buffer->bo;
610 uint32_t bo_offset = buffer->offset + offset;
611
612 genX(cmd_buffer_flush_state)(cmd_buffer);
613
614 /* TODO: We need to stomp base vertex to 0 somehow */
615 if (vs_prog_data->uses_basevertex || vs_prog_data->uses_baseinstance)
616 emit_base_vertex_instance_bo(cmd_buffer, bo, bo_offset + 12);
617
618 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
619 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
620 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
621 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, bo, bo_offset + 12);
622 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 16);
623
624 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
625 .IndirectParameterEnable = true,
626 .VertexAccessType = RANDOM,
627 .PrimitiveTopologyType = pipeline->topology);
628 }
629
630
631 void genX(CmdDispatch)(
632 VkCommandBuffer commandBuffer,
633 uint32_t x,
634 uint32_t y,
635 uint32_t z)
636 {
637 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
638 struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
639 const struct brw_cs_prog_data *prog_data = get_cs_prog_data(pipeline);
640
641 if (prog_data->uses_num_work_groups) {
642 struct anv_state state =
643 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, 12, 4);
644 uint32_t *sizes = state.map;
645 sizes[0] = x;
646 sizes[1] = y;
647 sizes[2] = z;
648 if (!cmd_buffer->device->info.has_llc)
649 anv_state_clflush(state);
650 cmd_buffer->state.num_workgroups_offset = state.offset;
651 cmd_buffer->state.num_workgroups_bo =
652 &cmd_buffer->device->dynamic_state_block_pool.bo;
653 }
654
655 genX(cmd_buffer_flush_compute_state)(cmd_buffer);
656
657 anv_batch_emit(&cmd_buffer->batch, GENX(GPGPU_WALKER),
658 .SIMDSize = prog_data->simd_size / 16,
659 .ThreadDepthCounterMaximum = 0,
660 .ThreadHeightCounterMaximum = 0,
661 .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max - 1,
662 .ThreadGroupIDXDimension = x,
663 .ThreadGroupIDYDimension = y,
664 .ThreadGroupIDZDimension = z,
665 .RightExecutionMask = pipeline->cs_right_mask,
666 .BottomExecutionMask = 0xffffffff);
667
668 anv_batch_emit(&cmd_buffer->batch, GENX(MEDIA_STATE_FLUSH));
669 }
670
671 #define GPGPU_DISPATCHDIMX 0x2500
672 #define GPGPU_DISPATCHDIMY 0x2504
673 #define GPGPU_DISPATCHDIMZ 0x2508
674
675 #define MI_PREDICATE_SRC0 0x2400
676 #define MI_PREDICATE_SRC1 0x2408
677
678 void genX(CmdDispatchIndirect)(
679 VkCommandBuffer commandBuffer,
680 VkBuffer _buffer,
681 VkDeviceSize offset)
682 {
683 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
684 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
685 struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
686 const struct brw_cs_prog_data *prog_data = get_cs_prog_data(pipeline);
687 struct anv_bo *bo = buffer->bo;
688 uint32_t bo_offset = buffer->offset + offset;
689 struct anv_batch *batch = &cmd_buffer->batch;
690
691 if (prog_data->uses_num_work_groups) {
692 cmd_buffer->state.num_workgroups_offset = bo_offset;
693 cmd_buffer->state.num_workgroups_bo = bo;
694 }
695
696 genX(cmd_buffer_flush_compute_state)(cmd_buffer);
697
698 emit_lrm(batch, GPGPU_DISPATCHDIMX, bo, bo_offset);
699 emit_lrm(batch, GPGPU_DISPATCHDIMY, bo, bo_offset + 4);
700 emit_lrm(batch, GPGPU_DISPATCHDIMZ, bo, bo_offset + 8);
701
702 #if GEN_GEN <= 7
703 /* Clear upper 32-bits of SRC0 and all 64-bits of SRC1 */
704 emit_lri(batch, MI_PREDICATE_SRC0 + 4, 0);
705 emit_lri(batch, MI_PREDICATE_SRC1 + 0, 0);
706 emit_lri(batch, MI_PREDICATE_SRC1 + 4, 0);
707
708 /* Load compute_dispatch_indirect_x_size into SRC0 */
709 emit_lrm(batch, MI_PREDICATE_SRC0, bo, bo_offset + 0);
710
711 /* predicate = (compute_dispatch_indirect_x_size == 0); */
712 anv_batch_emit(batch, GENX(MI_PREDICATE),
713 .LoadOperation = LOAD_LOAD,
714 .CombineOperation = COMBINE_SET,
715 .CompareOperation = COMPARE_SRCS_EQUAL);
716
717 /* Load compute_dispatch_indirect_y_size into SRC0 */
718 emit_lrm(batch, MI_PREDICATE_SRC0, bo, bo_offset + 4);
719
720 /* predicate |= (compute_dispatch_indirect_y_size == 0); */
721 anv_batch_emit(batch, GENX(MI_PREDICATE),
722 .LoadOperation = LOAD_LOAD,
723 .CombineOperation = COMBINE_OR,
724 .CompareOperation = COMPARE_SRCS_EQUAL);
725
726 /* Load compute_dispatch_indirect_z_size into SRC0 */
727 emit_lrm(batch, MI_PREDICATE_SRC0, bo, bo_offset + 8);
728
729 /* predicate |= (compute_dispatch_indirect_z_size == 0); */
730 anv_batch_emit(batch, GENX(MI_PREDICATE),
731 .LoadOperation = LOAD_LOAD,
732 .CombineOperation = COMBINE_OR,
733 .CompareOperation = COMPARE_SRCS_EQUAL);
734
735 /* predicate = !predicate; */
736 #define COMPARE_FALSE 1
737 anv_batch_emit(batch, GENX(MI_PREDICATE),
738 .LoadOperation = LOAD_LOADINV,
739 .CombineOperation = COMBINE_OR,
740 .CompareOperation = COMPARE_FALSE);
741 #endif
742
743 anv_batch_emit(batch, GENX(GPGPU_WALKER),
744 .IndirectParameterEnable = true,
745 .PredicateEnable = GEN_GEN <= 7,
746 .SIMDSize = prog_data->simd_size / 16,
747 .ThreadDepthCounterMaximum = 0,
748 .ThreadHeightCounterMaximum = 0,
749 .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max - 1,
750 .RightExecutionMask = pipeline->cs_right_mask,
751 .BottomExecutionMask = 0xffffffff);
752
753 anv_batch_emit(batch, GENX(MEDIA_STATE_FLUSH));
754 }
755
756 void
757 genX(flush_pipeline_select_3d)(struct anv_cmd_buffer *cmd_buffer)
758 {
759 if (cmd_buffer->state.current_pipeline != _3D) {
760 anv_batch_emit(&cmd_buffer->batch, GENX(PIPELINE_SELECT),
761 #if GEN_GEN >= 9
762 .MaskBits = 3,
763 #endif
764 .PipelineSelection = _3D);
765 cmd_buffer->state.current_pipeline = _3D;
766 }
767 }
768
769 struct anv_state
770 genX(cmd_buffer_alloc_null_surface_state)(struct anv_cmd_buffer *cmd_buffer,
771 struct anv_framebuffer *fb)
772 {
773 struct anv_state state =
774 anv_state_stream_alloc(&cmd_buffer->surface_state_stream, 64, 64);
775
776 struct GENX(RENDER_SURFACE_STATE) null_ss = {
777 .SurfaceType = SURFTYPE_NULL,
778 .SurfaceArray = fb->layers > 0,
779 .SurfaceFormat = ISL_FORMAT_R8G8B8A8_UNORM,
780 #if GEN_GEN >= 8
781 .TileMode = YMAJOR,
782 #else
783 .TiledSurface = true,
784 #endif
785 .Width = fb->width - 1,
786 .Height = fb->height - 1,
787 .Depth = fb->layers - 1,
788 .RenderTargetViewExtent = fb->layers - 1,
789 };
790
791 GENX(RENDER_SURFACE_STATE_pack)(NULL, state.map, &null_ss);
792
793 if (!cmd_buffer->device->info.has_llc)
794 anv_state_clflush(state);
795
796 return state;
797 }
798
799 static void
800 cmd_buffer_emit_depth_stencil(struct anv_cmd_buffer *cmd_buffer)
801 {
802 struct anv_device *device = cmd_buffer->device;
803 const struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
804 const struct anv_image_view *iview =
805 anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
806 const struct anv_image *image = iview ? iview->image : NULL;
807 const struct anv_format *anv_format =
808 iview ? anv_format_for_vk_format(iview->vk_format) : NULL;
809 const bool has_depth = iview && anv_format->has_depth;
810 const bool has_stencil = iview && anv_format->has_stencil;
811
812 /* FIXME: Implement the PMA stall W/A */
813 /* FIXME: Width and Height are wrong */
814
815 /* Emit 3DSTATE_DEPTH_BUFFER */
816 if (has_depth) {
817 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
818 .SurfaceType = SURFTYPE_2D,
819 .DepthWriteEnable = true,
820 .StencilWriteEnable = has_stencil,
821 .HierarchicalDepthBufferEnable = false,
822 .SurfaceFormat = isl_surf_get_depth_format(&device->isl_dev,
823 &image->depth_surface.isl),
824 .SurfacePitch = image->depth_surface.isl.row_pitch - 1,
825 .SurfaceBaseAddress = {
826 .bo = image->bo,
827 .offset = image->offset + image->depth_surface.offset,
828 },
829 .Height = fb->height - 1,
830 .Width = fb->width - 1,
831 .LOD = 0,
832 .Depth = 1 - 1,
833 .MinimumArrayElement = 0,
834 .DepthBufferObjectControlState = GENX(MOCS),
835 #if GEN_GEN >= 8
836 .SurfaceQPitch = isl_surf_get_array_pitch_el_rows(&image->depth_surface.isl) >> 2,
837 #endif
838 .RenderTargetViewExtent = 1 - 1);
839 } else {
840 /* Even when no depth buffer is present, the hardware requires that
841 * 3DSTATE_DEPTH_BUFFER be programmed correctly. The Broadwell PRM says:
842 *
843 * If a null depth buffer is bound, the driver must instead bind depth as:
844 * 3DSTATE_DEPTH.SurfaceType = SURFTYPE_2D
845 * 3DSTATE_DEPTH.Width = 1
846 * 3DSTATE_DEPTH.Height = 1
847 * 3DSTATE_DEPTH.SuraceFormat = D16_UNORM
848 * 3DSTATE_DEPTH.SurfaceBaseAddress = 0
849 * 3DSTATE_DEPTH.HierarchicalDepthBufferEnable = 0
850 * 3DSTATE_WM_DEPTH_STENCIL.DepthTestEnable = 0
851 * 3DSTATE_WM_DEPTH_STENCIL.DepthBufferWriteEnable = 0
852 *
853 * The PRM is wrong, though. The width and height must be programmed to
854 * actual framebuffer's width and height, even when neither depth buffer
855 * nor stencil buffer is present. Also, D16_UNORM is not allowed to
856 * be combined with a stencil buffer so we use D32_FLOAT instead.
857 */
858 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
859 .SurfaceType = SURFTYPE_2D,
860 .SurfaceFormat = D32_FLOAT,
861 .Width = fb->width - 1,
862 .Height = fb->height - 1,
863 .StencilWriteEnable = has_stencil);
864 }
865
866 /* Emit 3DSTATE_STENCIL_BUFFER */
867 if (has_stencil) {
868 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER),
869 #if GEN_GEN >= 8 || GEN_IS_HASWELL
870 .StencilBufferEnable = true,
871 #endif
872 .StencilBufferObjectControlState = GENX(MOCS),
873
874 /* Stencil buffers have strange pitch. The PRM says:
875 *
876 * The pitch must be set to 2x the value computed based on width,
877 * as the stencil buffer is stored with two rows interleaved.
878 */
879 .SurfacePitch = 2 * image->stencil_surface.isl.row_pitch - 1,
880
881 #if GEN_GEN >= 8
882 .SurfaceQPitch = isl_surf_get_array_pitch_el_rows(&image->stencil_surface.isl) >> 2,
883 #endif
884 .SurfaceBaseAddress = {
885 .bo = image->bo,
886 .offset = image->offset + image->stencil_surface.offset,
887 });
888 } else {
889 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER));
890 }
891
892 /* Disable hierarchial depth buffers. */
893 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_HIER_DEPTH_BUFFER));
894
895 /* Clear the clear params. */
896 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CLEAR_PARAMS));
897 }
898
899 /**
900 * @see anv_cmd_buffer_set_subpass()
901 */
902 void
903 genX(cmd_buffer_set_subpass)(struct anv_cmd_buffer *cmd_buffer,
904 struct anv_subpass *subpass)
905 {
906 cmd_buffer->state.subpass = subpass;
907
908 cmd_buffer->state.descriptors_dirty |= VK_SHADER_STAGE_FRAGMENT_BIT;
909
910 cmd_buffer_emit_depth_stencil(cmd_buffer);
911 }
912
913 void genX(CmdBeginRenderPass)(
914 VkCommandBuffer commandBuffer,
915 const VkRenderPassBeginInfo* pRenderPassBegin,
916 VkSubpassContents contents)
917 {
918 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
919 ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
920 ANV_FROM_HANDLE(anv_framebuffer, framebuffer, pRenderPassBegin->framebuffer);
921
922 cmd_buffer->state.framebuffer = framebuffer;
923 cmd_buffer->state.pass = pass;
924 anv_cmd_state_setup_attachments(cmd_buffer, pRenderPassBegin);
925
926 genX(flush_pipeline_select_3d)(cmd_buffer);
927
928 const VkRect2D *render_area = &pRenderPassBegin->renderArea;
929
930 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DRAWING_RECTANGLE),
931 .ClippedDrawingRectangleYMin = MAX2(render_area->offset.y, 0),
932 .ClippedDrawingRectangleXMin = MAX2(render_area->offset.x, 0),
933 .ClippedDrawingRectangleYMax =
934 render_area->offset.y + render_area->extent.height - 1,
935 .ClippedDrawingRectangleXMax =
936 render_area->offset.x + render_area->extent.width - 1,
937 .DrawingRectangleOriginY = 0,
938 .DrawingRectangleOriginX = 0);
939
940 genX(cmd_buffer_set_subpass)(cmd_buffer, pass->subpasses);
941 anv_cmd_buffer_clear_subpass(cmd_buffer);
942 }
943
944 void genX(CmdNextSubpass)(
945 VkCommandBuffer commandBuffer,
946 VkSubpassContents contents)
947 {
948 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
949
950 assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
951
952 anv_cmd_buffer_resolve_subpass(cmd_buffer);
953 genX(cmd_buffer_set_subpass)(cmd_buffer, cmd_buffer->state.subpass + 1);
954 anv_cmd_buffer_clear_subpass(cmd_buffer);
955 }
956
957 void genX(CmdEndRenderPass)(
958 VkCommandBuffer commandBuffer)
959 {
960 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
961
962 anv_cmd_buffer_resolve_subpass(cmd_buffer);
963 }
964
965 static void
966 emit_ps_depth_count(struct anv_batch *batch,
967 struct anv_bo *bo, uint32_t offset)
968 {
969 anv_batch_emit(batch, GENX(PIPE_CONTROL),
970 .DestinationAddressType = DAT_PPGTT,
971 .PostSyncOperation = WritePSDepthCount,
972 .DepthStallEnable = true,
973 .Address = { bo, offset });
974 }
975
976 static void
977 emit_query_availability(struct anv_batch *batch,
978 struct anv_bo *bo, uint32_t offset)
979 {
980 anv_batch_emit(batch, GENX(PIPE_CONTROL),
981 .DestinationAddressType = DAT_PPGTT,
982 .PostSyncOperation = WriteImmediateData,
983 .Address = { bo, offset },
984 .ImmediateData = 1);
985 }
986
987 void genX(CmdBeginQuery)(
988 VkCommandBuffer commandBuffer,
989 VkQueryPool queryPool,
990 uint32_t query,
991 VkQueryControlFlags flags)
992 {
993 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
994 ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
995
996 /* Workaround: When meta uses the pipeline with the VS disabled, it seems
997 * that the pipelining of the depth write breaks. What we see is that
998 * samples from the render pass clear leaks into the first query
999 * immediately after the clear. Doing a pipecontrol with a post-sync
1000 * operation and DepthStallEnable seems to work around the issue.
1001 */
1002 if (cmd_buffer->state.need_query_wa) {
1003 cmd_buffer->state.need_query_wa = false;
1004 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
1005 .DepthCacheFlushEnable = true,
1006 .DepthStallEnable = true);
1007 }
1008
1009 switch (pool->type) {
1010 case VK_QUERY_TYPE_OCCLUSION:
1011 emit_ps_depth_count(&cmd_buffer->batch, &pool->bo,
1012 query * sizeof(struct anv_query_pool_slot));
1013 break;
1014
1015 case VK_QUERY_TYPE_PIPELINE_STATISTICS:
1016 default:
1017 unreachable("");
1018 }
1019 }
1020
1021 void genX(CmdEndQuery)(
1022 VkCommandBuffer commandBuffer,
1023 VkQueryPool queryPool,
1024 uint32_t query)
1025 {
1026 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1027 ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
1028
1029 switch (pool->type) {
1030 case VK_QUERY_TYPE_OCCLUSION:
1031 emit_ps_depth_count(&cmd_buffer->batch, &pool->bo,
1032 query * sizeof(struct anv_query_pool_slot) + 8);
1033
1034 emit_query_availability(&cmd_buffer->batch, &pool->bo,
1035 query * sizeof(struct anv_query_pool_slot) + 16);
1036 break;
1037
1038 case VK_QUERY_TYPE_PIPELINE_STATISTICS:
1039 default:
1040 unreachable("");
1041 }
1042 }
1043
1044 #define TIMESTAMP 0x2358
1045
1046 void genX(CmdWriteTimestamp)(
1047 VkCommandBuffer commandBuffer,
1048 VkPipelineStageFlagBits pipelineStage,
1049 VkQueryPool queryPool,
1050 uint32_t query)
1051 {
1052 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1053 ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
1054 uint32_t offset = query * sizeof(struct anv_query_pool_slot);
1055
1056 assert(pool->type == VK_QUERY_TYPE_TIMESTAMP);
1057
1058 switch (pipelineStage) {
1059 case VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT:
1060 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM),
1061 .RegisterAddress = TIMESTAMP,
1062 .MemoryAddress = { &pool->bo, offset });
1063 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM),
1064 .RegisterAddress = TIMESTAMP + 4,
1065 .MemoryAddress = { &pool->bo, offset + 4 });
1066 break;
1067
1068 default:
1069 /* Everything else is bottom-of-pipe */
1070 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
1071 .DestinationAddressType = DAT_PPGTT,
1072 .PostSyncOperation = WriteTimestamp,
1073 .Address = { &pool->bo, offset });
1074 break;
1075 }
1076
1077 emit_query_availability(&cmd_buffer->batch, &pool->bo, query + 16);
1078 }
1079
1080 #if GEN_GEN > 7 || GEN_IS_HASWELL
1081
1082 #define alu_opcode(v) __gen_uint((v), 20, 31)
1083 #define alu_operand1(v) __gen_uint((v), 10, 19)
1084 #define alu_operand2(v) __gen_uint((v), 0, 9)
1085 #define alu(opcode, operand1, operand2) \
1086 alu_opcode(opcode) | alu_operand1(operand1) | alu_operand2(operand2)
1087
1088 #define OPCODE_NOOP 0x000
1089 #define OPCODE_LOAD 0x080
1090 #define OPCODE_LOADINV 0x480
1091 #define OPCODE_LOAD0 0x081
1092 #define OPCODE_LOAD1 0x481
1093 #define OPCODE_ADD 0x100
1094 #define OPCODE_SUB 0x101
1095 #define OPCODE_AND 0x102
1096 #define OPCODE_OR 0x103
1097 #define OPCODE_XOR 0x104
1098 #define OPCODE_STORE 0x180
1099 #define OPCODE_STOREINV 0x580
1100
1101 #define OPERAND_R0 0x00
1102 #define OPERAND_R1 0x01
1103 #define OPERAND_R2 0x02
1104 #define OPERAND_R3 0x03
1105 #define OPERAND_R4 0x04
1106 #define OPERAND_SRCA 0x20
1107 #define OPERAND_SRCB 0x21
1108 #define OPERAND_ACCU 0x31
1109 #define OPERAND_ZF 0x32
1110 #define OPERAND_CF 0x33
1111
1112 #define CS_GPR(n) (0x2600 + (n) * 8)
1113
1114 static void
1115 emit_load_alu_reg_u64(struct anv_batch *batch, uint32_t reg,
1116 struct anv_bo *bo, uint32_t offset)
1117 {
1118 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM),
1119 .RegisterAddress = reg,
1120 .MemoryAddress = { bo, offset });
1121 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM),
1122 .RegisterAddress = reg + 4,
1123 .MemoryAddress = { bo, offset + 4 });
1124 }
1125
1126 static void
1127 store_query_result(struct anv_batch *batch, uint32_t reg,
1128 struct anv_bo *bo, uint32_t offset, VkQueryResultFlags flags)
1129 {
1130 anv_batch_emit(batch, GENX(MI_STORE_REGISTER_MEM),
1131 .RegisterAddress = reg,
1132 .MemoryAddress = { bo, offset });
1133
1134 if (flags & VK_QUERY_RESULT_64_BIT)
1135 anv_batch_emit(batch, GENX(MI_STORE_REGISTER_MEM),
1136 .RegisterAddress = reg + 4,
1137 .MemoryAddress = { bo, offset + 4 });
1138 }
1139
1140 void genX(CmdCopyQueryPoolResults)(
1141 VkCommandBuffer commandBuffer,
1142 VkQueryPool queryPool,
1143 uint32_t firstQuery,
1144 uint32_t queryCount,
1145 VkBuffer destBuffer,
1146 VkDeviceSize destOffset,
1147 VkDeviceSize destStride,
1148 VkQueryResultFlags flags)
1149 {
1150 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
1151 ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
1152 ANV_FROM_HANDLE(anv_buffer, buffer, destBuffer);
1153 uint32_t slot_offset, dst_offset;
1154
1155 if (flags & VK_QUERY_RESULT_WAIT_BIT)
1156 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
1157 .CommandStreamerStallEnable = true,
1158 .StallAtPixelScoreboard = true);
1159
1160 dst_offset = buffer->offset + destOffset;
1161 for (uint32_t i = 0; i < queryCount; i++) {
1162
1163 slot_offset = (firstQuery + i) * sizeof(struct anv_query_pool_slot);
1164 switch (pool->type) {
1165 case VK_QUERY_TYPE_OCCLUSION:
1166 emit_load_alu_reg_u64(&cmd_buffer->batch,
1167 CS_GPR(0), &pool->bo, slot_offset);
1168 emit_load_alu_reg_u64(&cmd_buffer->batch,
1169 CS_GPR(1), &pool->bo, slot_offset + 8);
1170
1171 /* FIXME: We need to clamp the result for 32 bit. */
1172
1173 uint32_t *dw = anv_batch_emitn(&cmd_buffer->batch, 5, GENX(MI_MATH));
1174 dw[1] = alu(OPCODE_LOAD, OPERAND_SRCA, OPERAND_R1);
1175 dw[2] = alu(OPCODE_LOAD, OPERAND_SRCB, OPERAND_R0);
1176 dw[3] = alu(OPCODE_SUB, 0, 0);
1177 dw[4] = alu(OPCODE_STORE, OPERAND_R2, OPERAND_ACCU);
1178 break;
1179
1180 case VK_QUERY_TYPE_TIMESTAMP:
1181 emit_load_alu_reg_u64(&cmd_buffer->batch,
1182 CS_GPR(2), &pool->bo, slot_offset);
1183 break;
1184
1185 default:
1186 unreachable("unhandled query type");
1187 }
1188
1189 store_query_result(&cmd_buffer->batch,
1190 CS_GPR(2), buffer->bo, dst_offset, flags);
1191
1192 if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
1193 emit_load_alu_reg_u64(&cmd_buffer->batch, CS_GPR(0),
1194 &pool->bo, slot_offset + 16);
1195 if (flags & VK_QUERY_RESULT_64_BIT)
1196 store_query_result(&cmd_buffer->batch,
1197 CS_GPR(0), buffer->bo, dst_offset + 8, flags);
1198 else
1199 store_query_result(&cmd_buffer->batch,
1200 CS_GPR(0), buffer->bo, dst_offset + 4, flags);
1201 }
1202
1203 dst_offset += destStride;
1204 }
1205 }
1206
1207 #endif