projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
anv/icd: Advertise the right ABI version
[mesa.git] / src / vulkan / gen8_cmd_buffer.c
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <stdbool.h>
26 #include <string.h>
27 #include <unistd.h>
28 #include <fcntl.h>
29
30 #include "anv_private.h"
31
32 #include "gen8_pack.h"
33 #include "gen9_pack.h"
34
35 static void
36 cmd_buffer_flush_push_constants(struct anv_cmd_buffer *cmd_buffer)
37 {
38 static const uint32_t push_constant_opcodes[] = {
39 [MESA_SHADER_VERTEX] = 21,
40 [MESA_SHADER_TESS_CTRL] = 25, /* HS */
41 [MESA_SHADER_TESS_EVAL] = 26, /* DS */
42 [MESA_SHADER_GEOMETRY] = 22,
43 [MESA_SHADER_FRAGMENT] = 23,
44 [MESA_SHADER_COMPUTE] = 0,
45 };
46
47 VkShaderStageFlags flushed = 0;
48
49 anv_foreach_stage(stage, cmd_buffer->state.push_constants_dirty) {
50 if (stage == MESA_SHADER_COMPUTE)
51 continue;
52
53 struct anv_state state = anv_cmd_buffer_push_constants(cmd_buffer, stage);
54
55 if (state.offset == 0)
56 continue;
57
58 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CONSTANT_VS),
59 ._3DCommandSubOpcode = push_constant_opcodes[stage],
60 .ConstantBody = {
61 .PointerToConstantBuffer0 = { .offset = state.offset },
62 .ConstantBuffer0ReadLength = DIV_ROUND_UP(state.alloc_size, 32),
63 });
64
65 flushed |= mesa_to_vk_shader_stage(stage);
66 }
67
68 cmd_buffer->state.push_constants_dirty &= ~flushed;
69 }
70
71 #if ANV_GEN == 8
72 static void
73 emit_viewport_state(struct anv_cmd_buffer *cmd_buffer,
74 uint32_t count, const VkViewport *viewports)
75 {
76 struct anv_state sf_clip_state =
77 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, count * 64, 64);
78 struct anv_state cc_state =
79 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, count * 8, 32);
80
81 for (uint32_t i = 0; i < count; i++) {
82 const VkViewport *vp = &viewports[i];
83
84 /* The gen7 state struct has just the matrix and guardband fields, the
85 * gen8 struct adds the min/max viewport fields. */
86 struct GENX(SF_CLIP_VIEWPORT) sf_clip_viewport = {
87 .ViewportMatrixElementm00 = vp->width / 2,
88 .ViewportMatrixElementm11 = vp->height / 2,
89 .ViewportMatrixElementm22 = (vp->maxDepth - vp->minDepth) / 2,
90 .ViewportMatrixElementm30 = vp->x + vp->width / 2,
91 .ViewportMatrixElementm31 = vp->y + vp->height / 2,
92 .ViewportMatrixElementm32 = (vp->maxDepth + vp->minDepth) / 2,
93 .XMinClipGuardband = -1.0f,
94 .XMaxClipGuardband = 1.0f,
95 .YMinClipGuardband = -1.0f,
96 .YMaxClipGuardband = 1.0f,
97 .XMinViewPort = vp->x,
98 .XMaxViewPort = vp->x + vp->width - 1,
99 .YMinViewPort = vp->y,
100 .YMaxViewPort = vp->y + vp->height - 1,
101 };
102
103 struct GENX(CC_VIEWPORT) cc_viewport = {
104 .MinimumDepth = vp->minDepth,
105 .MaximumDepth = vp->maxDepth
106 };
107
108 GENX(SF_CLIP_VIEWPORT_pack)(NULL, sf_clip_state.map + i * 64,
109 &sf_clip_viewport);
110 GENX(CC_VIEWPORT_pack)(NULL, cc_state.map + i * 32, &cc_viewport);
111 }
112
113 if (!cmd_buffer->device->info.has_llc) {
114 anv_state_clflush(sf_clip_state);
115 anv_state_clflush(cc_state);
116 }
117
118 anv_batch_emit(&cmd_buffer->batch,
119 GENX(3DSTATE_VIEWPORT_STATE_POINTERS_CC),
120 .CCViewportPointer = cc_state.offset);
121 anv_batch_emit(&cmd_buffer->batch,
122 GENX(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP),
123 .SFClipViewportPointer = sf_clip_state.offset);
124 }
125
126 void
127 gen8_cmd_buffer_emit_viewport(struct anv_cmd_buffer *cmd_buffer)
128 {
129 if (cmd_buffer->state.dynamic.viewport.count > 0) {
130 emit_viewport_state(cmd_buffer, cmd_buffer->state.dynamic.viewport.count,
131 cmd_buffer->state.dynamic.viewport.viewports);
132 } else {
133 /* If viewport count is 0, this is taken to mean "use the default" */
134 emit_viewport_state(cmd_buffer, 1,
135 &(VkViewport) {
136 .x = 0.0f,
137 .y = 0.0f,
138 .width = cmd_buffer->state.framebuffer->width,
139 .height = cmd_buffer->state.framebuffer->height,
140 .minDepth = 0.0f,
141 .maxDepth = 1.0f,
142 });
143 }
144 }
145 #endif
146
147 static void
148 cmd_buffer_flush_state(struct anv_cmd_buffer *cmd_buffer)
149 {
150 struct anv_pipeline *pipeline = cmd_buffer->state.pipeline;
151 uint32_t *p;
152
153 uint32_t vb_emit = cmd_buffer->state.vb_dirty & pipeline->vb_used;
154
155 assert((pipeline->active_stages & VK_SHADER_STAGE_COMPUTE_BIT) == 0);
156
157 if (cmd_buffer->state.current_pipeline != _3D) {
158 anv_batch_emit(&cmd_buffer->batch, GENX(PIPELINE_SELECT),
159 #if ANV_GEN >= 9
160 .MaskBits = 3,
161 #endif
162 .PipelineSelection = _3D);
163 cmd_buffer->state.current_pipeline = _3D;
164 }
165
166 if (vb_emit) {
167 const uint32_t num_buffers = __builtin_popcount(vb_emit);
168 const uint32_t num_dwords = 1 + num_buffers * 4;
169
170 p = anv_batch_emitn(&cmd_buffer->batch, num_dwords,
171 GENX(3DSTATE_VERTEX_BUFFERS));
172 uint32_t vb, i = 0;
173 for_each_bit(vb, vb_emit) {
174 struct anv_buffer *buffer = cmd_buffer->state.vertex_bindings[vb].buffer;
175 uint32_t offset = cmd_buffer->state.vertex_bindings[vb].offset;
176
177 struct GENX(VERTEX_BUFFER_STATE) state = {
178 .VertexBufferIndex = vb,
179 .MemoryObjectControlState = GENX(MOCS),
180 .AddressModifyEnable = true,
181 .BufferPitch = pipeline->binding_stride[vb],
182 .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
183 .BufferSize = buffer->size - offset
184 };
185
186 GENX(VERTEX_BUFFER_STATE_pack)(&cmd_buffer->batch, &p[1 + i * 4], &state);
187 i++;
188 }
189 }
190
191 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_PIPELINE) {
192 /* If somebody compiled a pipeline after starting a command buffer the
193 * scratch bo may have grown since we started this cmd buffer (and
194 * emitted STATE_BASE_ADDRESS). If we're binding that pipeline now,
195 * reemit STATE_BASE_ADDRESS so that we use the bigger scratch bo. */
196 if (cmd_buffer->state.scratch_size < pipeline->total_scratch)
197 anv_cmd_buffer_emit_state_base_address(cmd_buffer);
198
199 anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
200 }
201
202 #if ANV_GEN >= 9
203 /* On SKL+ the new constants don't take effect until the next corresponding
204 * 3DSTATE_BINDING_TABLE_POINTER_* command is parsed so we need to ensure
205 * that is sent. As it is, we re-emit binding tables but we could hold on
206 * to the offset of the most recent binding table and only re-emit the
207 * 3DSTATE_BINDING_TABLE_POINTER_* command.
208 */
209 cmd_buffer->state.descriptors_dirty |=
210 cmd_buffer->state.push_constants_dirty &
211 cmd_buffer->state.pipeline->active_stages;
212 #endif
213
214 if (cmd_buffer->state.descriptors_dirty)
215 gen7_cmd_buffer_flush_descriptor_sets(cmd_buffer);
216
217 if (cmd_buffer->state.push_constants_dirty)
218 cmd_buffer_flush_push_constants(cmd_buffer);
219
220 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_VIEWPORT)
221 gen8_cmd_buffer_emit_viewport(cmd_buffer);
222
223 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_SCISSOR)
224 gen7_cmd_buffer_emit_scissor(cmd_buffer);
225
226 if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
227 ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH)) {
228 uint32_t sf_dw[GENX(3DSTATE_SF_length)];
229 struct GENX(3DSTATE_SF) sf = {
230 GENX(3DSTATE_SF_header),
231 .LineWidth = cmd_buffer->state.dynamic.line_width,
232 };
233 GENX(3DSTATE_SF_pack)(NULL, sf_dw, &sf);
234 /* FIXME: gen9.fs */
235 anv_batch_emit_merge(&cmd_buffer->batch, sf_dw, pipeline->gen8.sf);
236 }
237
238 if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
239 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS)){
240 bool enable_bias = cmd_buffer->state.dynamic.depth_bias.bias != 0.0f ||
241 cmd_buffer->state.dynamic.depth_bias.slope != 0.0f;
242
243 uint32_t raster_dw[GENX(3DSTATE_RASTER_length)];
244 struct GENX(3DSTATE_RASTER) raster = {
245 GENX(3DSTATE_RASTER_header),
246 .GlobalDepthOffsetEnableSolid = enable_bias,
247 .GlobalDepthOffsetEnableWireframe = enable_bias,
248 .GlobalDepthOffsetEnablePoint = enable_bias,
249 .GlobalDepthOffsetConstant = cmd_buffer->state.dynamic.depth_bias.bias,
250 .GlobalDepthOffsetScale = cmd_buffer->state.dynamic.depth_bias.slope,
251 .GlobalDepthOffsetClamp = cmd_buffer->state.dynamic.depth_bias.clamp
252 };
253 GENX(3DSTATE_RASTER_pack)(NULL, raster_dw, &raster);
254 anv_batch_emit_merge(&cmd_buffer->batch, raster_dw,
255 pipeline->gen8.raster);
256 }
257
258 /* Stencil reference values moved from COLOR_CALC_STATE in gen8 to
259 * 3DSTATE_WM_DEPTH_STENCIL in gen9. That means the dirty bits gets split
260 * across different state packets for gen8 and gen9. We handle that by
261 * using a big old #if switch here.
262 */
263 #if ANV_GEN == 8
264 if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS |
265 ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE)) {
266 struct anv_state cc_state =
267 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer,
268 GEN8_COLOR_CALC_STATE_length, 64);
269 struct GEN8_COLOR_CALC_STATE cc = {
270 .BlendConstantColorRed = cmd_buffer->state.dynamic.blend_constants[0],
271 .BlendConstantColorGreen = cmd_buffer->state.dynamic.blend_constants[1],
272 .BlendConstantColorBlue = cmd_buffer->state.dynamic.blend_constants[2],
273 .BlendConstantColorAlpha = cmd_buffer->state.dynamic.blend_constants[3],
274 .StencilReferenceValue =
275 cmd_buffer->state.dynamic.stencil_reference.front,
276 .BackFaceStencilReferenceValue =
277 cmd_buffer->state.dynamic.stencil_reference.back,
278 };
279 GEN8_COLOR_CALC_STATE_pack(NULL, cc_state.map, &cc);
280
281 if (!cmd_buffer->device->info.has_llc)
282 anv_state_clflush(cc_state);
283
284 anv_batch_emit(&cmd_buffer->batch,
285 GEN8_3DSTATE_CC_STATE_POINTERS,
286 .ColorCalcStatePointer = cc_state.offset,
287 .ColorCalcStatePointerValid = true);
288 }
289
290 if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
291 ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK |
292 ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK)) {
293 uint32_t wm_depth_stencil_dw[GEN8_3DSTATE_WM_DEPTH_STENCIL_length];
294
295 struct GEN8_3DSTATE_WM_DEPTH_STENCIL wm_depth_stencil = {
296 GEN8_3DSTATE_WM_DEPTH_STENCIL_header,
297
298 /* Is this what we need to do? */
299 .StencilBufferWriteEnable =
300 cmd_buffer->state.dynamic.stencil_write_mask.front != 0,
301
302 .StencilTestMask =
303 cmd_buffer->state.dynamic.stencil_compare_mask.front & 0xff,
304 .StencilWriteMask =
305 cmd_buffer->state.dynamic.stencil_write_mask.front & 0xff,
306
307 .BackfaceStencilTestMask =
308 cmd_buffer->state.dynamic.stencil_compare_mask.back & 0xff,
309 .BackfaceStencilWriteMask =
310 cmd_buffer->state.dynamic.stencil_write_mask.back & 0xff,
311 };
312 GEN8_3DSTATE_WM_DEPTH_STENCIL_pack(NULL, wm_depth_stencil_dw,
313 &wm_depth_stencil);
314
315 anv_batch_emit_merge(&cmd_buffer->batch, wm_depth_stencil_dw,
316 pipeline->gen8.wm_depth_stencil);
317 }
318 #else
319 if (cmd_buffer->state.dirty & ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS) {
320 struct anv_state cc_state =
321 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer,
322 GEN9_COLOR_CALC_STATE_length, 64);
323 struct GEN9_COLOR_CALC_STATE cc = {
324 .BlendConstantColorRed = cmd_buffer->state.dynamic.blend_constants[0],
325 .BlendConstantColorGreen = cmd_buffer->state.dynamic.blend_constants[1],
326 .BlendConstantColorBlue = cmd_buffer->state.dynamic.blend_constants[2],
327 .BlendConstantColorAlpha = cmd_buffer->state.dynamic.blend_constants[3],
328 };
329 GEN9_COLOR_CALC_STATE_pack(NULL, cc_state.map, &cc);
330
331 if (!cmd_buffer->device->info.has_llc)
332 anv_state_clflush(cc_state);
333
334 anv_batch_emit(&cmd_buffer->batch,
335 GEN9_3DSTATE_CC_STATE_POINTERS,
336 .ColorCalcStatePointer = cc_state.offset,
337 .ColorCalcStatePointerValid = true);
338 }
339
340 if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
341 ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK |
342 ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK |
343 ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE)) {
344 uint32_t dwords[GEN9_3DSTATE_WM_DEPTH_STENCIL_length];
345 struct anv_dynamic_state *d = &cmd_buffer->state.dynamic;
346 struct GEN9_3DSTATE_WM_DEPTH_STENCIL wm_depth_stencil = {
347 GEN9_3DSTATE_WM_DEPTH_STENCIL_header,
348
349 .StencilBufferWriteEnable = d->stencil_write_mask.front != 0,
350
351 .StencilTestMask = d->stencil_compare_mask.front & 0xff,
352 .StencilWriteMask = d->stencil_write_mask.front & 0xff,
353
354 .BackfaceStencilTestMask = d->stencil_compare_mask.back & 0xff,
355 .BackfaceStencilWriteMask = d->stencil_write_mask.back & 0xff,
356
357 .StencilReferenceValue = d->stencil_reference.front,
358 .BackfaceStencilReferenceValue = d->stencil_reference.back
359 };
360 GEN9_3DSTATE_WM_DEPTH_STENCIL_pack(NULL, dwords, &wm_depth_stencil);
361
362 anv_batch_emit_merge(&cmd_buffer->batch, dwords,
363 pipeline->gen9.wm_depth_stencil);
364 }
365 #endif
366
367 if (cmd_buffer->state.dirty & (ANV_CMD_DIRTY_PIPELINE |
368 ANV_CMD_DIRTY_INDEX_BUFFER)) {
369 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_VF),
370 .IndexedDrawCutIndexEnable = pipeline->primitive_restart,
371 .CutIndex = cmd_buffer->state.restart_index,
372 );
373 }
374
375 cmd_buffer->state.vb_dirty &= ~vb_emit;
376 cmd_buffer->state.dirty = 0;
377 }
378
379 void genX(CmdDraw)(
380 VkCommandBuffer commandBuffer,
381 uint32_t vertexCount,
382 uint32_t instanceCount,
383 uint32_t firstVertex,
384 uint32_t firstInstance)
385 {
386 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
387
388 cmd_buffer_flush_state(cmd_buffer);
389
390 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
391 .VertexAccessType = SEQUENTIAL,
392 .VertexCountPerInstance = vertexCount,
393 .StartVertexLocation = firstVertex,
394 .InstanceCount = instanceCount,
395 .StartInstanceLocation = firstInstance,
396 .BaseVertexLocation = 0);
397 }
398
399 void genX(CmdDrawIndexed)(
400 VkCommandBuffer commandBuffer,
401 uint32_t indexCount,
402 uint32_t instanceCount,
403 uint32_t firstIndex,
404 int32_t vertexOffset,
405 uint32_t firstInstance)
406 {
407 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
408
409 cmd_buffer_flush_state(cmd_buffer);
410
411 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
412 .VertexAccessType = RANDOM,
413 .VertexCountPerInstance = indexCount,
414 .StartVertexLocation = firstIndex,
415 .InstanceCount = instanceCount,
416 .StartInstanceLocation = firstInstance,
417 .BaseVertexLocation = vertexOffset);
418 }
419
420 static void
421 emit_lrm(struct anv_batch *batch,
422 uint32_t reg, struct anv_bo *bo, uint32_t offset)
423 {
424 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM),
425 .RegisterAddress = reg,
426 .MemoryAddress = { bo, offset });
427 }
428
429 static void
430 emit_lri(struct anv_batch *batch, uint32_t reg, uint32_t imm)
431 {
432 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM),
433 .RegisterOffset = reg,
434 .DataDWord = imm);
435 }
436
437 /* Auto-Draw / Indirect Registers */
438 #define GEN7_3DPRIM_END_OFFSET 0x2420
439 #define GEN7_3DPRIM_START_VERTEX 0x2430
440 #define GEN7_3DPRIM_VERTEX_COUNT 0x2434
441 #define GEN7_3DPRIM_INSTANCE_COUNT 0x2438
442 #define GEN7_3DPRIM_START_INSTANCE 0x243C
443 #define GEN7_3DPRIM_BASE_VERTEX 0x2440
444
445 void genX(CmdDrawIndirect)(
446 VkCommandBuffer commandBuffer,
447 VkBuffer _buffer,
448 VkDeviceSize offset,
449 uint32_t drawCount,
450 uint32_t stride)
451 {
452 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
453 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
454 struct anv_bo *bo = buffer->bo;
455 uint32_t bo_offset = buffer->offset + offset;
456
457 cmd_buffer_flush_state(cmd_buffer);
458
459 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
460 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
461 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
462 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 12);
463 emit_lri(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, 0);
464
465 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
466 .IndirectParameterEnable = true,
467 .VertexAccessType = SEQUENTIAL);
468 }
469
470 void genX(CmdBindIndexBuffer)(
471 VkCommandBuffer commandBuffer,
472 VkBuffer _buffer,
473 VkDeviceSize offset,
474 VkIndexType indexType)
475 {
476 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
477 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
478
479 static const uint32_t vk_to_gen_index_type[] = {
480 [VK_INDEX_TYPE_UINT16] = INDEX_WORD,
481 [VK_INDEX_TYPE_UINT32] = INDEX_DWORD,
482 };
483
484 static const uint32_t restart_index_for_type[] = {
485 [VK_INDEX_TYPE_UINT16] = UINT16_MAX,
486 [VK_INDEX_TYPE_UINT32] = UINT32_MAX,
487 };
488
489 cmd_buffer->state.restart_index = restart_index_for_type[indexType];
490
491 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_INDEX_BUFFER),
492 .IndexFormat = vk_to_gen_index_type[indexType],
493 .MemoryObjectControlState = GENX(MOCS),
494 .BufferStartingAddress = { buffer->bo, buffer->offset + offset },
495 .BufferSize = buffer->size - offset);
496
497 cmd_buffer->state.dirty |= ANV_CMD_DIRTY_INDEX_BUFFER;
498 }
499
500 static VkResult
501 flush_compute_descriptor_set(struct anv_cmd_buffer *cmd_buffer)
502 {
503 struct anv_device *device = cmd_buffer->device;
504 struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
505 struct anv_state surfaces = { 0, }, samplers = { 0, };
506 VkResult result;
507
508 result = anv_cmd_buffer_emit_samplers(cmd_buffer,
509 MESA_SHADER_COMPUTE, &samplers);
510 if (result != VK_SUCCESS)
511 return result;
512 result = anv_cmd_buffer_emit_binding_table(cmd_buffer,
513 MESA_SHADER_COMPUTE, &surfaces);
514 if (result != VK_SUCCESS)
515 return result;
516
517 struct anv_state push_state = anv_cmd_buffer_cs_push_constants(cmd_buffer);
518
519 const struct brw_cs_prog_data *cs_prog_data = &pipeline->cs_prog_data;
520 const struct brw_stage_prog_data *prog_data = &cs_prog_data->base;
521
522 unsigned local_id_dwords = cs_prog_data->local_invocation_id_regs * 8;
523 unsigned push_constant_data_size =
524 (prog_data->nr_params + local_id_dwords) * sizeof(gl_constant_value);
525 unsigned reg_aligned_constant_size = ALIGN(push_constant_data_size, 32);
526 unsigned push_constant_regs = reg_aligned_constant_size / 32;
527
528 anv_batch_emit(&cmd_buffer->batch, GENX(MEDIA_CURBE_LOAD),
529 .CURBETotalDataLength = push_state.alloc_size,
530 .CURBEDataStartAddress = push_state.offset);
531
532 struct anv_state state =
533 anv_state_pool_emit(&device->dynamic_state_pool,
534 GENX(INTERFACE_DESCRIPTOR_DATA), 64,
535 .KernelStartPointer = pipeline->cs_simd,
536 .KernelStartPointerHigh = 0,
537 .BindingTablePointer = surfaces.offset,
538 .BindingTableEntryCount = 0,
539 .SamplerStatePointer = samplers.offset,
540 .SamplerCount = 0,
541 .ConstantIndirectURBEntryReadLength = push_constant_regs,
542 .ConstantURBEntryReadOffset = 0,
543 .NumberofThreadsinGPGPUThreadGroup = 0);
544
545 uint32_t size = GENX(INTERFACE_DESCRIPTOR_DATA_length) * sizeof(uint32_t);
546 anv_batch_emit(&cmd_buffer->batch, GENX(MEDIA_INTERFACE_DESCRIPTOR_LOAD),
547 .InterfaceDescriptorTotalLength = size,
548 .InterfaceDescriptorDataStartAddress = state.offset);
549
550 return VK_SUCCESS;
551 }
552
553 static void
554 cmd_buffer_flush_compute_state(struct anv_cmd_buffer *cmd_buffer)
555 {
556 struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
557 VkResult result;
558
559 assert(pipeline->active_stages == VK_SHADER_STAGE_COMPUTE_BIT);
560
561 if (cmd_buffer->state.current_pipeline != GPGPU) {
562 anv_batch_emit(&cmd_buffer->batch, GENX(PIPELINE_SELECT),
563 #if ANV_GEN >= 9
564 .MaskBits = 3,
565 #endif
566 .PipelineSelection = GPGPU);
567 cmd_buffer->state.current_pipeline = GPGPU;
568 }
569
570 if (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)
571 anv_batch_emit_batch(&cmd_buffer->batch, &pipeline->batch);
572
573 if ((cmd_buffer->state.descriptors_dirty & VK_SHADER_STAGE_COMPUTE_BIT) ||
574 (cmd_buffer->state.compute_dirty & ANV_CMD_DIRTY_PIPELINE)) {
575 result = flush_compute_descriptor_set(cmd_buffer);
576 assert(result == VK_SUCCESS);
577 cmd_buffer->state.descriptors_dirty &= ~VK_SHADER_STAGE_COMPUTE_BIT;
578 }
579
580 cmd_buffer->state.compute_dirty = 0;
581 }
582
583 void genX(CmdDrawIndexedIndirect)(
584 VkCommandBuffer commandBuffer,
585 VkBuffer _buffer,
586 VkDeviceSize offset,
587 uint32_t drawCount,
588 uint32_t stride)
589 {
590 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
591 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
592 struct anv_bo *bo = buffer->bo;
593 uint32_t bo_offset = buffer->offset + offset;
594
595 cmd_buffer_flush_state(cmd_buffer);
596
597 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_VERTEX_COUNT, bo, bo_offset);
598 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_INSTANCE_COUNT, bo, bo_offset + 4);
599 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_VERTEX, bo, bo_offset + 8);
600 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_BASE_VERTEX, bo, bo_offset + 12);
601 emit_lrm(&cmd_buffer->batch, GEN7_3DPRIM_START_INSTANCE, bo, bo_offset + 16);
602
603 anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE),
604 .IndirectParameterEnable = true,
605 .VertexAccessType = RANDOM);
606 }
607
608 void genX(CmdDispatch)(
609 VkCommandBuffer commandBuffer,
610 uint32_t x,
611 uint32_t y,
612 uint32_t z)
613 {
614 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
615 struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
616 struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
617
618 cmd_buffer_flush_compute_state(cmd_buffer);
619
620 anv_batch_emit(&cmd_buffer->batch, GENX(GPGPU_WALKER),
621 .SIMDSize = prog_data->simd_size / 16,
622 .ThreadDepthCounterMaximum = 0,
623 .ThreadHeightCounterMaximum = 0,
624 .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max - 1,
625 .ThreadGroupIDXDimension = x,
626 .ThreadGroupIDYDimension = y,
627 .ThreadGroupIDZDimension = z,
628 .RightExecutionMask = pipeline->cs_right_mask,
629 .BottomExecutionMask = 0xffffffff);
630
631 anv_batch_emit(&cmd_buffer->batch, GENX(MEDIA_STATE_FLUSH));
632 }
633
634 #define GPGPU_DISPATCHDIMX 0x2500
635 #define GPGPU_DISPATCHDIMY 0x2504
636 #define GPGPU_DISPATCHDIMZ 0x2508
637
638 void genX(CmdDispatchIndirect)(
639 VkCommandBuffer commandBuffer,
640 VkBuffer _buffer,
641 VkDeviceSize offset)
642 {
643 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
644 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
645 struct anv_pipeline *pipeline = cmd_buffer->state.compute_pipeline;
646 struct brw_cs_prog_data *prog_data = &pipeline->cs_prog_data;
647 struct anv_bo *bo = buffer->bo;
648 uint32_t bo_offset = buffer->offset + offset;
649
650 cmd_buffer_flush_compute_state(cmd_buffer);
651
652 emit_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMX, bo, bo_offset);
653 emit_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMY, bo, bo_offset + 4);
654 emit_lrm(&cmd_buffer->batch, GPGPU_DISPATCHDIMZ, bo, bo_offset + 8);
655
656 anv_batch_emit(&cmd_buffer->batch, GENX(GPGPU_WALKER),
657 .IndirectParameterEnable = true,
658 .SIMDSize = prog_data->simd_size / 16,
659 .ThreadDepthCounterMaximum = 0,
660 .ThreadHeightCounterMaximum = 0,
661 .ThreadWidthCounterMaximum = pipeline->cs_thread_width_max - 1,
662 .RightExecutionMask = pipeline->cs_right_mask,
663 .BottomExecutionMask = 0xffffffff);
664
665 anv_batch_emit(&cmd_buffer->batch, GENX(MEDIA_STATE_FLUSH));
666 }
667
668 static void
669 cmd_buffer_emit_depth_stencil(struct anv_cmd_buffer *cmd_buffer)
670 {
671 const struct anv_framebuffer *fb = cmd_buffer->state.framebuffer;
672 const struct anv_image_view *iview =
673 anv_cmd_buffer_get_depth_stencil_view(cmd_buffer);
674 const struct anv_image *image = iview ? iview->image : NULL;
675 const bool has_depth = iview && iview->format->depth_format;
676 const bool has_stencil = iview && iview->format->has_stencil;
677
678 /* FIXME: Implement the PMA stall W/A */
679 /* FIXME: Width and Height are wrong */
680
681 /* Emit 3DSTATE_DEPTH_BUFFER */
682 if (has_depth) {
683 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
684 .SurfaceType = SURFTYPE_2D,
685 .DepthWriteEnable = iview->format->depth_format,
686 .StencilWriteEnable = has_stencil,
687 .HierarchicalDepthBufferEnable = false,
688 .SurfaceFormat = iview->format->depth_format,
689 .SurfacePitch = image->depth_surface.isl.row_pitch - 1,
690 .SurfaceBaseAddress = {
691 .bo = image->bo,
692 .offset = image->depth_surface.offset,
693 },
694 .Height = fb->height - 1,
695 .Width = fb->width - 1,
696 .LOD = 0,
697 .Depth = 1 - 1,
698 .MinimumArrayElement = 0,
699 .DepthBufferObjectControlState = GENX(MOCS),
700 .RenderTargetViewExtent = 1 - 1,
701 .SurfaceQPitch = isl_surf_get_array_pitch_el_rows(&image->depth_surface.isl) >> 2);
702 } else {
703 /* Even when no depth buffer is present, the hardware requires that
704 * 3DSTATE_DEPTH_BUFFER be programmed correctly. The Broadwell PRM says:
705 *
706 * If a null depth buffer is bound, the driver must instead bind depth as:
707 * 3DSTATE_DEPTH.SurfaceType = SURFTYPE_2D
708 * 3DSTATE_DEPTH.Width = 1
709 * 3DSTATE_DEPTH.Height = 1
710 * 3DSTATE_DEPTH.SuraceFormat = D16_UNORM
711 * 3DSTATE_DEPTH.SurfaceBaseAddress = 0
712 * 3DSTATE_DEPTH.HierarchicalDepthBufferEnable = 0
713 * 3DSTATE_WM_DEPTH_STENCIL.DepthTestEnable = 0
714 * 3DSTATE_WM_DEPTH_STENCIL.DepthBufferWriteEnable = 0
715 *
716 * The PRM is wrong, though. The width and height must be programmed to
717 * actual framebuffer's width and height, even when neither depth buffer
718 * nor stencil buffer is present.
719 */
720 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DEPTH_BUFFER),
721 .SurfaceType = SURFTYPE_2D,
722 .SurfaceFormat = D16_UNORM,
723 .Width = fb->width - 1,
724 .Height = fb->height - 1,
725 .StencilWriteEnable = has_stencil);
726 }
727
728 /* Emit 3DSTATE_STENCIL_BUFFER */
729 if (has_stencil) {
730 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER),
731 .StencilBufferEnable = true,
732 .StencilBufferObjectControlState = GENX(MOCS),
733
734 /* Stencil buffers have strange pitch. The PRM says:
735 *
736 * The pitch must be set to 2x the value computed based on width,
737 * as the stencil buffer is stored with two rows interleaved.
738 */
739 .SurfacePitch = 2 * image->stencil_surface.isl.row_pitch - 1,
740
741 .SurfaceBaseAddress = {
742 .bo = image->bo,
743 .offset = image->offset + image->stencil_surface.offset,
744 },
745 .SurfaceQPitch = isl_surf_get_array_pitch_el_rows(&image->stencil_surface.isl) >> 2);
746 } else {
747 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STENCIL_BUFFER));
748 }
749
750 /* Disable hierarchial depth buffers. */
751 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_HIER_DEPTH_BUFFER));
752
753 /* Clear the clear params. */
754 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_CLEAR_PARAMS));
755 }
756
757 void
758 genX(cmd_buffer_begin_subpass)(struct anv_cmd_buffer *cmd_buffer,
759 struct anv_subpass *subpass)
760 {
761 cmd_buffer->state.subpass = subpass;
762
763 cmd_buffer->state.descriptors_dirty |= VK_SHADER_STAGE_FRAGMENT_BIT;
764
765 cmd_buffer_emit_depth_stencil(cmd_buffer);
766 }
767
768 void genX(CmdBeginRenderPass)(
769 VkCommandBuffer commandBuffer,
770 const VkRenderPassBeginInfo* pRenderPassBegin,
771 VkSubpassContents contents)
772 {
773 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
774 ANV_FROM_HANDLE(anv_render_pass, pass, pRenderPassBegin->renderPass);
775 ANV_FROM_HANDLE(anv_framebuffer, framebuffer, pRenderPassBegin->framebuffer);
776
777 cmd_buffer->state.framebuffer = framebuffer;
778 cmd_buffer->state.pass = pass;
779
780 const VkRect2D *render_area = &pRenderPassBegin->renderArea;
781
782 anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DRAWING_RECTANGLE),
783 .ClippedDrawingRectangleYMin = render_area->offset.y,
784 .ClippedDrawingRectangleXMin = render_area->offset.x,
785 .ClippedDrawingRectangleYMax =
786 render_area->offset.y + render_area->extent.height - 1,
787 .ClippedDrawingRectangleXMax =
788 render_area->offset.x + render_area->extent.width - 1,
789 .DrawingRectangleOriginY = 0,
790 .DrawingRectangleOriginX = 0);
791
792 anv_cmd_buffer_clear_attachments(cmd_buffer, pass,
793 pRenderPassBegin->pClearValues);
794
795 genX(cmd_buffer_begin_subpass)(cmd_buffer, pass->subpasses);
796 }
797
798 void genX(CmdNextSubpass)(
799 VkCommandBuffer commandBuffer,
800 VkSubpassContents contents)
801 {
802 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
803
804 assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
805
806 genX(cmd_buffer_begin_subpass)(cmd_buffer, cmd_buffer->state.subpass + 1);
807 }
808
809 void genX(CmdEndRenderPass)(
810 VkCommandBuffer commandBuffer)
811 {
812 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
813
814 /* Emit a flushing pipe control at the end of a pass. This is kind of a
815 * hack but it ensures that render targets always actually get written.
816 * Eventually, we should do flushing based on image format transitions
817 * or something of that nature.
818 */
819 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
820 .PostSyncOperation = NoWrite,
821 .RenderTargetCacheFlushEnable = true,
822 .InstructionCacheInvalidateEnable = true,
823 .DepthCacheFlushEnable = true,
824 .VFCacheInvalidationEnable = true,
825 .TextureCacheInvalidationEnable = true,
826 .CommandStreamerStallEnable = true);
827 }
828
829 static void
830 emit_ps_depth_count(struct anv_batch *batch,
831 struct anv_bo *bo, uint32_t offset)
832 {
833 anv_batch_emit(batch, GENX(PIPE_CONTROL),
834 .DestinationAddressType = DAT_PPGTT,
835 .PostSyncOperation = WritePSDepthCount,
836 .Address = { bo, offset }); /* FIXME: This is only lower 32 bits */
837 }
838
839 void genX(CmdBeginQuery)(
840 VkCommandBuffer commandBuffer,
841 VkQueryPool queryPool,
842 uint32_t entry,
843 VkQueryControlFlags flags)
844 {
845 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
846 ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
847
848 switch (pool->type) {
849 case VK_QUERY_TYPE_OCCLUSION:
850 emit_ps_depth_count(&cmd_buffer->batch, &pool->bo,
851 entry * sizeof(struct anv_query_pool_slot));
852 break;
853
854 case VK_QUERY_TYPE_PIPELINE_STATISTICS:
855 default:
856 unreachable("");
857 }
858 }
859
860 void genX(CmdEndQuery)(
861 VkCommandBuffer commandBuffer,
862 VkQueryPool queryPool,
863 uint32_t entry)
864 {
865 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
866 ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
867
868 switch (pool->type) {
869 case VK_QUERY_TYPE_OCCLUSION:
870 emit_ps_depth_count(&cmd_buffer->batch, &pool->bo,
871 entry * sizeof(struct anv_query_pool_slot) + 8);
872 break;
873
874 case VK_QUERY_TYPE_PIPELINE_STATISTICS:
875 default:
876 unreachable("");
877 }
878 }
879
880 #define TIMESTAMP 0x2358
881
882 void genX(CmdWriteTimestamp)(
883 VkCommandBuffer commandBuffer,
884 VkPipelineStageFlagBits pipelineStage,
885 VkQueryPool queryPool,
886 uint32_t entry)
887 {
888 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
889 ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
890
891 assert(pool->type == VK_QUERY_TYPE_TIMESTAMP);
892
893 switch (pipelineStage) {
894 case VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT:
895 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM),
896 .RegisterAddress = TIMESTAMP,
897 .MemoryAddress = { &pool->bo, entry * 8 });
898 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM),
899 .RegisterAddress = TIMESTAMP + 4,
900 .MemoryAddress = { &pool->bo, entry * 8 + 4 });
901 break;
902
903 default:
904 /* Everything else is bottom-of-pipe */
905 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
906 .DestinationAddressType = DAT_PPGTT,
907 .PostSyncOperation = WriteTimestamp,
908 .Address = /* FIXME: This is only lower 32 bits */
909 { &pool->bo, entry * 8 });
910 break;
911 }
912 }
913
914 #define alu_opcode(v) __gen_field((v), 20, 31)
915 #define alu_operand1(v) __gen_field((v), 10, 19)
916 #define alu_operand2(v) __gen_field((v), 0, 9)
917 #define alu(opcode, operand1, operand2) \
918 alu_opcode(opcode) | alu_operand1(operand1) | alu_operand2(operand2)
919
920 #define OPCODE_NOOP 0x000
921 #define OPCODE_LOAD 0x080
922 #define OPCODE_LOADINV 0x480
923 #define OPCODE_LOAD0 0x081
924 #define OPCODE_LOAD1 0x481
925 #define OPCODE_ADD 0x100
926 #define OPCODE_SUB 0x101
927 #define OPCODE_AND 0x102
928 #define OPCODE_OR 0x103
929 #define OPCODE_XOR 0x104
930 #define OPCODE_STORE 0x180
931 #define OPCODE_STOREINV 0x580
932
933 #define OPERAND_R0 0x00
934 #define OPERAND_R1 0x01
935 #define OPERAND_R2 0x02
936 #define OPERAND_R3 0x03
937 #define OPERAND_R4 0x04
938 #define OPERAND_SRCA 0x20
939 #define OPERAND_SRCB 0x21
940 #define OPERAND_ACCU 0x31
941 #define OPERAND_ZF 0x32
942 #define OPERAND_CF 0x33
943
944 #define CS_GPR(n) (0x2600 + (n) * 8)
945
946 static void
947 emit_load_alu_reg_u64(struct anv_batch *batch, uint32_t reg,
948 struct anv_bo *bo, uint32_t offset)
949 {
950 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM),
951 .RegisterAddress = reg,
952 .MemoryAddress = { bo, offset });
953 anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM),
954 .RegisterAddress = reg + 4,
955 .MemoryAddress = { bo, offset + 4 });
956 }
957
958 void genX(CmdCopyQueryPoolResults)(
959 VkCommandBuffer commandBuffer,
960 VkQueryPool queryPool,
961 uint32_t startQuery,
962 uint32_t queryCount,
963 VkBuffer destBuffer,
964 VkDeviceSize destOffset,
965 VkDeviceSize destStride,
966 VkQueryResultFlags flags)
967 {
968 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
969 ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
970 ANV_FROM_HANDLE(anv_buffer, buffer, destBuffer);
971 uint32_t slot_offset, dst_offset;
972
973 if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
974 /* Where is the availabilty info supposed to go? */
975 anv_finishme("VK_QUERY_RESULT_WITH_AVAILABILITY_BIT");
976 return;
977 }
978
979 assert(pool->type == VK_QUERY_TYPE_OCCLUSION);
980
981 /* FIXME: If we're not waiting, should we just do this on the CPU? */
982 if (flags & VK_QUERY_RESULT_WAIT_BIT)
983 anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL),
984 .CommandStreamerStallEnable = true,
985 .StallAtPixelScoreboard = true);
986
987 dst_offset = buffer->offset + destOffset;
988 for (uint32_t i = 0; i < queryCount; i++) {
989
990 slot_offset = (startQuery + i) * sizeof(struct anv_query_pool_slot);
991
992 emit_load_alu_reg_u64(&cmd_buffer->batch, CS_GPR(0), &pool->bo, slot_offset);
993 emit_load_alu_reg_u64(&cmd_buffer->batch, CS_GPR(1), &pool->bo, slot_offset + 8);
994
995 /* FIXME: We need to clamp the result for 32 bit. */
996
997 uint32_t *dw = anv_batch_emitn(&cmd_buffer->batch, 5, GENX(MI_MATH));
998 dw[1] = alu(OPCODE_LOAD, OPERAND_SRCA, OPERAND_R1);
999 dw[2] = alu(OPCODE_LOAD, OPERAND_SRCB, OPERAND_R0);
1000 dw[3] = alu(OPCODE_SUB, 0, 0);
1001 dw[4] = alu(OPCODE_STORE, OPERAND_R2, OPERAND_ACCU);
1002
1003 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM),
1004 .RegisterAddress = CS_GPR(2),
1005 /* FIXME: This is only lower 32 bits */
1006 .MemoryAddress = { buffer->bo, dst_offset });
1007
1008 if (flags & VK_QUERY_RESULT_64_BIT)
1009 anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM),
1010 .RegisterAddress = CS_GPR(2) + 4,
1011 /* FIXME: This is only lower 32 bits */
1012 .MemoryAddress = { buffer->bo, dst_offset + 4 });
1013
1014 dst_offset += destStride;
1015 }
1016 }