2 * Copyright © 2017 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
26 * ============================= GENXML CODE =============================
27 * [This file is compiled once per generation.]
28 * =======================================================================
30 * Query object support. This allows measuring various simple statistics
31 * via counters on the GPU. We use GenX code for MI_MATH calculations.
36 #include "perf/gen_perf.h"
37 #include "pipe/p_defines.h"
38 #include "pipe/p_state.h"
39 #include "pipe/p_context.h"
40 #include "pipe/p_screen.h"
41 #include "util/u_inlines.h"
42 #include "util/u_upload_mgr.h"
43 #include "iris_context.h"
44 #include "iris_defines.h"
45 #include "iris_fence.h"
46 #include "iris_monitor.h"
47 #include "iris_resource.h"
48 #include "iris_screen.h"
50 #include "iris_genx_macros.h"
52 #define SO_PRIM_STORAGE_NEEDED(n) (GENX(SO_PRIM_STORAGE_NEEDED0_num) + (n) * 8)
53 #define SO_NUM_PRIMS_WRITTEN(n) (GENX(SO_NUM_PRIMS_WRITTEN0_num) + (n) * 8)
56 enum pipe_query_type type
;
65 struct iris_state_ref query_state_ref
;
66 struct iris_query_snapshots
*map
;
67 struct iris_syncpt
*syncpt
;
71 struct iris_monitor_object
*monitor
;
74 struct iris_query_snapshots
{
75 /** iris_render_condition's saved MI_PREDICATE_RESULT value. */
76 uint64_t predicate_result
;
78 /** Have the start/end snapshots landed? */
79 uint64_t snapshots_landed
;
81 /** Starting and ending counter snapshots */
86 struct iris_query_so_overflow
{
87 uint64_t predicate_result
;
88 uint64_t snapshots_landed
;
91 uint64_t prim_storage_needed
[2];
92 uint64_t num_prims
[2];
96 static struct gen_mi_value
97 query_mem64(struct iris_query
*q
, uint32_t offset
)
99 struct iris_address addr
= {
100 .bo
= iris_resource_bo(q
->query_state_ref
.res
),
101 .offset
= q
->query_state_ref
.offset
+ offset
,
104 return gen_mi_mem64(addr
);
108 * Is this type of query written by PIPE_CONTROL?
111 iris_is_query_pipelined(struct iris_query
*q
)
114 case PIPE_QUERY_OCCLUSION_COUNTER
:
115 case PIPE_QUERY_OCCLUSION_PREDICATE
:
116 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE
:
117 case PIPE_QUERY_TIMESTAMP
:
118 case PIPE_QUERY_TIMESTAMP_DISJOINT
:
119 case PIPE_QUERY_TIME_ELAPSED
:
128 mark_available(struct iris_context
*ice
, struct iris_query
*q
)
130 struct iris_batch
*batch
= &ice
->batches
[q
->batch_idx
];
131 unsigned flags
= PIPE_CONTROL_WRITE_IMMEDIATE
;
132 unsigned offset
= offsetof(struct iris_query_snapshots
, snapshots_landed
);
133 struct iris_bo
*bo
= iris_resource_bo(q
->query_state_ref
.res
);
134 offset
+= q
->query_state_ref
.offset
;
136 if (!iris_is_query_pipelined(q
)) {
137 ice
->vtbl
.store_data_imm64(batch
, bo
, offset
, true);
139 /* Order available *after* the query results. */
140 flags
|= PIPE_CONTROL_FLUSH_ENABLE
;
141 iris_emit_pipe_control_write(batch
, "query: mark available",
142 flags
, bo
, offset
, true);
147 * Write PS_DEPTH_COUNT to q->(dest) via a PIPE_CONTROL.
150 iris_pipelined_write(struct iris_batch
*batch
,
151 struct iris_query
*q
,
152 enum pipe_control_flags flags
,
155 const struct gen_device_info
*devinfo
= &batch
->screen
->devinfo
;
156 const unsigned optional_cs_stall
=
157 GEN_GEN
== 9 && devinfo
->gt
== 4 ? PIPE_CONTROL_CS_STALL
: 0;
158 struct iris_bo
*bo
= iris_resource_bo(q
->query_state_ref
.res
);
160 iris_emit_pipe_control_write(batch
, "query: pipelined snapshot write",
161 flags
| optional_cs_stall
,
166 write_value(struct iris_context
*ice
, struct iris_query
*q
, unsigned offset
)
168 struct iris_batch
*batch
= &ice
->batches
[q
->batch_idx
];
169 struct iris_bo
*bo
= iris_resource_bo(q
->query_state_ref
.res
);
171 if (!iris_is_query_pipelined(q
)) {
172 iris_emit_pipe_control_flush(batch
,
173 "query: non-pipelined snapshot write",
174 PIPE_CONTROL_CS_STALL
|
175 PIPE_CONTROL_STALL_AT_SCOREBOARD
);
180 case PIPE_QUERY_OCCLUSION_COUNTER
:
181 case PIPE_QUERY_OCCLUSION_PREDICATE
:
182 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE
:
184 /* "Driver must program PIPE_CONTROL with only Depth Stall Enable
185 * bit set prior to programming a PIPE_CONTROL with Write PS Depth
186 * Count sync operation."
188 iris_emit_pipe_control_flush(batch
,
189 "workaround: depth stall before writing "
191 PIPE_CONTROL_DEPTH_STALL
);
193 iris_pipelined_write(&ice
->batches
[IRIS_BATCH_RENDER
], q
,
194 PIPE_CONTROL_WRITE_DEPTH_COUNT
|
195 PIPE_CONTROL_DEPTH_STALL
,
198 case PIPE_QUERY_TIME_ELAPSED
:
199 case PIPE_QUERY_TIMESTAMP
:
200 case PIPE_QUERY_TIMESTAMP_DISJOINT
:
201 iris_pipelined_write(&ice
->batches
[IRIS_BATCH_RENDER
], q
,
202 PIPE_CONTROL_WRITE_TIMESTAMP
,
205 case PIPE_QUERY_PRIMITIVES_GENERATED
:
206 ice
->vtbl
.store_register_mem64(batch
,
208 GENX(CL_INVOCATION_COUNT_num
) :
209 SO_PRIM_STORAGE_NEEDED(q
->index
),
212 case PIPE_QUERY_PRIMITIVES_EMITTED
:
213 ice
->vtbl
.store_register_mem64(batch
,
214 SO_NUM_PRIMS_WRITTEN(q
->index
),
217 case PIPE_QUERY_PIPELINE_STATISTICS_SINGLE
: {
218 static const uint32_t index_to_reg
[] = {
219 GENX(IA_VERTICES_COUNT_num
),
220 GENX(IA_PRIMITIVES_COUNT_num
),
221 GENX(VS_INVOCATION_COUNT_num
),
222 GENX(GS_INVOCATION_COUNT_num
),
223 GENX(GS_PRIMITIVES_COUNT_num
),
224 GENX(CL_INVOCATION_COUNT_num
),
225 GENX(CL_PRIMITIVES_COUNT_num
),
226 GENX(PS_INVOCATION_COUNT_num
),
227 GENX(HS_INVOCATION_COUNT_num
),
228 GENX(DS_INVOCATION_COUNT_num
),
229 GENX(CS_INVOCATION_COUNT_num
),
231 const uint32_t reg
= index_to_reg
[q
->index
];
233 ice
->vtbl
.store_register_mem64(batch
, reg
, bo
, offset
, false);
242 write_overflow_values(struct iris_context
*ice
, struct iris_query
*q
, bool end
)
244 struct iris_batch
*batch
= &ice
->batches
[IRIS_BATCH_RENDER
];
245 uint32_t count
= q
->type
== PIPE_QUERY_SO_OVERFLOW_PREDICATE
? 1 : 4;
246 struct iris_bo
*bo
= iris_resource_bo(q
->query_state_ref
.res
);
247 uint32_t offset
= q
->query_state_ref
.offset
;
249 iris_emit_pipe_control_flush(batch
,
250 "query: write SO overflow snapshots",
251 PIPE_CONTROL_CS_STALL
|
252 PIPE_CONTROL_STALL_AT_SCOREBOARD
);
253 for (uint32_t i
= 0; i
< count
; i
++) {
254 int s
= q
->index
+ i
;
255 int g_idx
= offset
+ offsetof(struct iris_query_so_overflow
,
256 stream
[s
].num_prims
[end
]);
257 int w_idx
= offset
+ offsetof(struct iris_query_so_overflow
,
258 stream
[s
].prim_storage_needed
[end
]);
259 ice
->vtbl
.store_register_mem64(batch
, SO_NUM_PRIMS_WRITTEN(s
),
261 ice
->vtbl
.store_register_mem64(batch
, SO_PRIM_STORAGE_NEEDED(s
),
267 iris_raw_timestamp_delta(uint64_t time0
, uint64_t time1
)
270 return (1ULL << TIMESTAMP_BITS
) + time1
- time0
;
272 return time1
- time0
;
277 stream_overflowed(struct iris_query_so_overflow
*so
, int s
)
279 return (so
->stream
[s
].prim_storage_needed
[1] -
280 so
->stream
[s
].prim_storage_needed
[0]) !=
281 (so
->stream
[s
].num_prims
[1] - so
->stream
[s
].num_prims
[0]);
285 calculate_result_on_cpu(const struct gen_device_info
*devinfo
,
286 struct iris_query
*q
)
289 case PIPE_QUERY_OCCLUSION_PREDICATE
:
290 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE
:
291 q
->result
= q
->map
->end
!= q
->map
->start
;
293 case PIPE_QUERY_TIMESTAMP
:
294 case PIPE_QUERY_TIMESTAMP_DISJOINT
:
295 /* The timestamp is the single starting snapshot. */
296 q
->result
= gen_device_info_timebase_scale(devinfo
, q
->map
->start
);
297 q
->result
&= (1ull << TIMESTAMP_BITS
) - 1;
299 case PIPE_QUERY_TIME_ELAPSED
:
300 q
->result
= iris_raw_timestamp_delta(q
->map
->start
, q
->map
->end
);
301 q
->result
= gen_device_info_timebase_scale(devinfo
, q
->result
);
302 q
->result
&= (1ull << TIMESTAMP_BITS
) - 1;
304 case PIPE_QUERY_SO_OVERFLOW_PREDICATE
:
305 q
->result
= stream_overflowed((void *) q
->map
, q
->index
);
307 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE
:
309 for (int i
= 0; i
< MAX_VERTEX_STREAMS
; i
++)
310 q
->result
|= stream_overflowed((void *) q
->map
, i
);
312 case PIPE_QUERY_PIPELINE_STATISTICS_SINGLE
:
313 q
->result
= q
->map
->end
- q
->map
->start
;
315 /* WaDividePSInvocationCountBy4:HSW,BDW */
316 if (GEN_GEN
== 8 && q
->index
== PIPE_STAT_QUERY_PS_INVOCATIONS
)
319 case PIPE_QUERY_OCCLUSION_COUNTER
:
320 case PIPE_QUERY_PRIMITIVES_GENERATED
:
321 case PIPE_QUERY_PRIMITIVES_EMITTED
:
323 q
->result
= q
->map
->end
- q
->map
->start
;
331 * Calculate the streamout overflow for stream \p idx:
333 * (num_prims[1] - num_prims[0]) - (storage_needed[1] - storage_needed[0])
335 static struct gen_mi_value
336 calc_overflow_for_stream(struct gen_mi_builder
*b
,
337 struct iris_query
*q
,
340 #define C(counter, i) query_mem64(q, \
341 offsetof(struct iris_query_so_overflow, stream[idx].counter[i]))
343 return gen_mi_isub(b
, gen_mi_isub(b
, C(num_prims
, 1), C(num_prims
, 0)),
344 gen_mi_isub(b
, C(prim_storage_needed
, 1),
345 C(prim_storage_needed
, 0)));
350 * Calculate whether any stream has overflowed.
352 static struct gen_mi_value
353 calc_overflow_any_stream(struct gen_mi_builder
*b
, struct iris_query
*q
)
355 struct gen_mi_value stream_result
[MAX_VERTEX_STREAMS
];
356 for (int i
= 0; i
< MAX_VERTEX_STREAMS
; i
++)
357 stream_result
[i
] = calc_overflow_for_stream(b
, q
, i
);
359 struct gen_mi_value result
= stream_result
[0];
360 for (int i
= 1; i
< MAX_VERTEX_STREAMS
; i
++)
361 result
= gen_mi_ior(b
, result
, stream_result
[i
]);
367 query_is_boolean(enum pipe_query_type type
)
370 case PIPE_QUERY_OCCLUSION_PREDICATE
:
371 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE
:
372 case PIPE_QUERY_SO_OVERFLOW_PREDICATE
:
373 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE
:
381 * Calculate the result using MI_MATH.
383 static struct gen_mi_value
384 calculate_result_on_gpu(const struct gen_device_info
*devinfo
,
385 struct gen_mi_builder
*b
,
386 struct iris_query
*q
)
388 struct gen_mi_value result
;
389 struct gen_mi_value start_val
=
390 query_mem64(q
, offsetof(struct iris_query_snapshots
, start
));
391 struct gen_mi_value end_val
=
392 query_mem64(q
, offsetof(struct iris_query_snapshots
, end
));
395 case PIPE_QUERY_SO_OVERFLOW_PREDICATE
:
396 result
= calc_overflow_for_stream(b
, q
, q
->index
);
398 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE
:
399 result
= calc_overflow_any_stream(b
, q
);
401 case PIPE_QUERY_TIMESTAMP
: {
402 /* TODO: This discards any fractional bits of the timebase scale.
403 * We would need to do a bit of fixed point math on the CS ALU, or
404 * launch an actual shader to calculate this with full precision.
406 uint32_t scale
= 1000000000ull / devinfo
->timestamp_frequency
;
407 result
= gen_mi_iand(b
, gen_mi_imm((1ull << 36) - 1),
408 gen_mi_imul_imm(b
, start_val
, scale
));
411 case PIPE_QUERY_TIME_ELAPSED
: {
412 /* TODO: This discards fractional bits (see above). */
413 uint32_t scale
= 1000000000ull / devinfo
->timestamp_frequency
;
414 result
= gen_mi_imul_imm(b
, gen_mi_isub(b
, end_val
, start_val
), scale
);
418 result
= gen_mi_isub(b
, end_val
, start_val
);
422 /* WaDividePSInvocationCountBy4:HSW,BDW */
424 q
->type
== PIPE_QUERY_PIPELINE_STATISTICS_SINGLE
&&
425 q
->index
== PIPE_STAT_QUERY_PS_INVOCATIONS
)
426 result
= gen_mi_ushr32_imm(b
, result
, 2);
428 if (query_is_boolean(q
->type
))
429 result
= gen_mi_iand(b
, gen_mi_nz(b
, result
), gen_mi_imm(1));
434 static struct pipe_query
*
435 iris_create_query(struct pipe_context
*ctx
,
439 struct iris_query
*q
= calloc(1, sizeof(struct iris_query
));
441 q
->type
= query_type
;
445 if (q
->type
== PIPE_QUERY_PIPELINE_STATISTICS_SINGLE
&&
446 q
->index
== PIPE_STAT_QUERY_CS_INVOCATIONS
)
447 q
->batch_idx
= IRIS_BATCH_COMPUTE
;
449 q
->batch_idx
= IRIS_BATCH_RENDER
;
450 return (struct pipe_query
*) q
;
453 static struct pipe_query
*
454 iris_create_batch_query(struct pipe_context
*ctx
,
455 unsigned num_queries
,
456 unsigned *query_types
)
458 struct iris_context
*ice
= (void *) ctx
;
459 struct iris_query
*q
= calloc(1, sizeof(struct iris_query
));
462 q
->type
= PIPE_QUERY_DRIVER_SPECIFIC
;
464 q
->monitor
= iris_create_monitor_object(ice
, num_queries
, query_types
);
465 if (unlikely(!q
->monitor
)) {
470 return (struct pipe_query
*) q
;
474 iris_destroy_query(struct pipe_context
*ctx
, struct pipe_query
*p_query
)
476 struct iris_query
*query
= (void *) p_query
;
477 struct iris_screen
*screen
= (void *) ctx
->screen
;
478 if (query
->monitor
) {
479 iris_destroy_monitor_object(ctx
, query
->monitor
);
480 query
->monitor
= NULL
;
482 iris_syncpt_reference(screen
, &query
->syncpt
, NULL
);
489 iris_begin_query(struct pipe_context
*ctx
, struct pipe_query
*query
)
491 struct iris_context
*ice
= (void *) ctx
;
492 struct iris_query
*q
= (void *) query
;
495 return iris_begin_monitor(ctx
, q
->monitor
);
500 if (q
->type
== PIPE_QUERY_SO_OVERFLOW_PREDICATE
||
501 q
->type
== PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE
)
502 size
= sizeof(struct iris_query_so_overflow
);
504 size
= sizeof(struct iris_query_snapshots
);
506 u_upload_alloc(ice
->query_buffer_uploader
, 0,
507 size
, size
, &q
->query_state_ref
.offset
,
508 &q
->query_state_ref
.res
, &ptr
);
510 if (!iris_resource_bo(q
->query_state_ref
.res
))
519 WRITE_ONCE(q
->map
->snapshots_landed
, false);
521 if (q
->type
== PIPE_QUERY_PRIMITIVES_GENERATED
&& q
->index
== 0) {
522 ice
->state
.prims_generated_query_active
= true;
523 ice
->state
.dirty
|= IRIS_DIRTY_STREAMOUT
| IRIS_DIRTY_CLIP
;
526 if (q
->type
== PIPE_QUERY_SO_OVERFLOW_PREDICATE
||
527 q
->type
== PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE
)
528 write_overflow_values(ice
, q
, false);
531 q
->query_state_ref
.offset
+
532 offsetof(struct iris_query_snapshots
, start
));
538 iris_end_query(struct pipe_context
*ctx
, struct pipe_query
*query
)
540 struct iris_context
*ice
= (void *) ctx
;
541 struct iris_query
*q
= (void *) query
;
544 return iris_end_monitor(ctx
, q
->monitor
);
546 struct iris_batch
*batch
= &ice
->batches
[q
->batch_idx
];
548 if (q
->type
== PIPE_QUERY_TIMESTAMP
) {
549 iris_begin_query(ctx
, query
);
550 iris_batch_reference_signal_syncpt(batch
, &q
->syncpt
);
551 mark_available(ice
, q
);
555 if (q
->type
== PIPE_QUERY_PRIMITIVES_GENERATED
&& q
->index
== 0) {
556 ice
->state
.prims_generated_query_active
= false;
557 ice
->state
.dirty
|= IRIS_DIRTY_STREAMOUT
| IRIS_DIRTY_CLIP
;
560 if (q
->type
== PIPE_QUERY_SO_OVERFLOW_PREDICATE
||
561 q
->type
== PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE
)
562 write_overflow_values(ice
, q
, true);
565 q
->query_state_ref
.offset
+
566 offsetof(struct iris_query_snapshots
, end
));
568 iris_batch_reference_signal_syncpt(batch
, &q
->syncpt
);
569 mark_available(ice
, q
);
575 * See if the snapshots have landed for a query, and if so, compute the
576 * result and mark it ready. Does not flush (unlike iris_get_query_result).
579 iris_check_query_no_flush(struct iris_context
*ice
, struct iris_query
*q
)
581 struct iris_screen
*screen
= (void *) ice
->ctx
.screen
;
582 const struct gen_device_info
*devinfo
= &screen
->devinfo
;
584 if (!q
->ready
&& READ_ONCE(q
->map
->snapshots_landed
)) {
585 calculate_result_on_cpu(devinfo
, q
);
590 iris_get_query_result(struct pipe_context
*ctx
,
591 struct pipe_query
*query
,
593 union pipe_query_result
*result
)
595 struct iris_context
*ice
= (void *) ctx
;
596 struct iris_query
*q
= (void *) query
;
599 return iris_get_monitor_result(ctx
, q
->monitor
, wait
, result
->batch
);
601 struct iris_screen
*screen
= (void *) ctx
->screen
;
602 const struct gen_device_info
*devinfo
= &screen
->devinfo
;
604 if (unlikely(screen
->no_hw
)) {
610 struct iris_batch
*batch
= &ice
->batches
[q
->batch_idx
];
611 if (q
->syncpt
== iris_batch_get_signal_syncpt(batch
))
612 iris_batch_flush(batch
);
614 while (!READ_ONCE(q
->map
->snapshots_landed
)) {
616 iris_wait_syncpt(ctx
->screen
, q
->syncpt
, INT64_MAX
);
621 assert(READ_ONCE(q
->map
->snapshots_landed
));
622 calculate_result_on_cpu(devinfo
, q
);
627 result
->u64
= q
->result
;
633 iris_get_query_result_resource(struct pipe_context
*ctx
,
634 struct pipe_query
*query
,
636 enum pipe_query_value_type result_type
,
638 struct pipe_resource
*p_res
,
641 struct iris_context
*ice
= (void *) ctx
;
642 struct iris_query
*q
= (void *) query
;
643 struct iris_batch
*batch
= &ice
->batches
[q
->batch_idx
];
644 const struct gen_device_info
*devinfo
= &batch
->screen
->devinfo
;
645 struct iris_resource
*res
= (void *) p_res
;
646 struct iris_bo
*query_bo
= iris_resource_bo(q
->query_state_ref
.res
);
647 struct iris_bo
*dst_bo
= iris_resource_bo(p_res
);
648 unsigned snapshots_landed_offset
=
649 offsetof(struct iris_query_snapshots
, snapshots_landed
);
651 res
->bind_history
|= PIPE_BIND_QUERY_BUFFER
;
654 /* They're asking for the availability of the result. If we still
655 * have commands queued up which produce the result, submit them
656 * now so that progress happens. Either way, copy the snapshots
657 * landed field to the destination resource.
659 if (q
->syncpt
== iris_batch_get_signal_syncpt(batch
))
660 iris_batch_flush(batch
);
662 ice
->vtbl
.copy_mem_mem(batch
, dst_bo
, offset
,
663 query_bo
, snapshots_landed_offset
,
664 result_type
<= PIPE_QUERY_TYPE_U32
? 4 : 8);
668 if (!q
->ready
&& READ_ONCE(q
->map
->snapshots_landed
)) {
669 /* The final snapshots happen to have landed, so let's just compute
670 * the result on the CPU now...
672 calculate_result_on_cpu(devinfo
, q
);
676 /* We happen to have the result on the CPU, so just copy it. */
677 if (result_type
<= PIPE_QUERY_TYPE_U32
) {
678 ice
->vtbl
.store_data_imm32(batch
, dst_bo
, offset
, q
->result
);
680 ice
->vtbl
.store_data_imm64(batch
, dst_bo
, offset
, q
->result
);
683 /* Make sure the result lands before they use bind the QBO elsewhere
684 * and use the result.
686 // XXX: Why? i965 doesn't do this.
687 iris_emit_pipe_control_flush(batch
,
688 "query: unknown QBO flushing hack",
689 PIPE_CONTROL_CS_STALL
);
693 bool predicated
= !wait
&& !q
->stalled
;
695 struct gen_mi_builder b
;
696 gen_mi_builder_init(&b
, batch
);
698 struct gen_mi_value result
= calculate_result_on_gpu(devinfo
, &b
, q
);
699 struct gen_mi_value dst
=
700 result_type
<= PIPE_QUERY_TYPE_U32
? gen_mi_mem32(rw_bo(dst_bo
, offset
))
701 : gen_mi_mem64(rw_bo(dst_bo
, offset
));
704 gen_mi_store(&b
, gen_mi_reg32(MI_PREDICATE_RESULT
),
705 gen_mi_mem64(ro_bo(query_bo
, snapshots_landed_offset
)));
706 gen_mi_store_if(&b
, dst
, result
);
708 gen_mi_store(&b
, dst
, result
);
713 iris_set_active_query_state(struct pipe_context
*ctx
, bool enable
)
715 struct iris_context
*ice
= (void *) ctx
;
717 if (ice
->state
.statistics_counters_enabled
== enable
)
720 // XXX: most packets aren't paying attention to this yet, because it'd
721 // have to be done dynamically at draw time, which is a pain
722 ice
->state
.statistics_counters_enabled
= enable
;
723 ice
->state
.dirty
|= IRIS_DIRTY_CLIP
|
726 IRIS_DIRTY_STREAMOUT
|
734 set_predicate_enable(struct iris_context
*ice
, bool value
)
737 ice
->state
.predicate
= IRIS_PREDICATE_STATE_RENDER
;
739 ice
->state
.predicate
= IRIS_PREDICATE_STATE_DONT_RENDER
;
743 set_predicate_for_result(struct iris_context
*ice
,
744 struct iris_query
*q
,
747 struct iris_batch
*batch
= &ice
->batches
[IRIS_BATCH_RENDER
];
748 struct iris_bo
*bo
= iris_resource_bo(q
->query_state_ref
.res
);
750 /* The CPU doesn't have the query result yet; use hardware predication */
751 ice
->state
.predicate
= IRIS_PREDICATE_STATE_USE_BIT
;
753 /* Ensure the memory is coherent for MI_LOAD_REGISTER_* commands. */
754 iris_emit_pipe_control_flush(batch
,
755 "conditional rendering: set predicate",
756 PIPE_CONTROL_FLUSH_ENABLE
);
759 struct gen_mi_builder b
;
760 gen_mi_builder_init(&b
, batch
);
762 struct gen_mi_value result
;
765 case PIPE_QUERY_SO_OVERFLOW_PREDICATE
:
766 result
= calc_overflow_for_stream(&b
, q
, q
->index
);
768 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE
:
769 result
= calc_overflow_any_stream(&b
, q
);
772 /* PIPE_QUERY_OCCLUSION_* */
773 struct gen_mi_value start
=
774 query_mem64(q
, offsetof(struct iris_query_snapshots
, start
));
775 struct gen_mi_value end
=
776 query_mem64(q
, offsetof(struct iris_query_snapshots
, end
));
777 result
= gen_mi_isub(&b
, end
, start
);
782 result
= inverted
? gen_mi_z(&b
, result
) : gen_mi_nz(&b
, result
);
783 result
= gen_mi_iand(&b
, result
, gen_mi_imm(1));
785 /* We immediately set the predicate on the render batch, as all the
786 * counters come from 3D operations. However, we may need to predicate
787 * a compute dispatch, which executes in a different GEM context and has
788 * a different MI_PREDICATE_RESULT register. So, we save the result to
789 * memory and reload it in iris_launch_grid.
791 gen_mi_value_ref(&b
, result
);
792 gen_mi_store(&b
, gen_mi_reg32(MI_PREDICATE_RESULT
), result
);
793 gen_mi_store(&b
, query_mem64(q
, offsetof(struct iris_query_snapshots
,
794 predicate_result
)), result
);
795 ice
->state
.compute_predicate
= bo
;
799 iris_render_condition(struct pipe_context
*ctx
,
800 struct pipe_query
*query
,
802 enum pipe_render_cond_flag mode
)
804 struct iris_context
*ice
= (void *) ctx
;
805 struct iris_query
*q
= (void *) query
;
807 /* The old condition isn't relevant; we'll update it if necessary */
808 ice
->state
.compute_predicate
= NULL
;
809 ice
->condition
.query
= q
;
810 ice
->condition
.condition
= condition
;
813 ice
->state
.predicate
= IRIS_PREDICATE_STATE_RENDER
;
817 iris_check_query_no_flush(ice
, q
);
819 if (q
->result
|| q
->ready
) {
820 set_predicate_enable(ice
, (q
->result
!= 0) ^ condition
);
822 if (mode
== PIPE_RENDER_COND_NO_WAIT
||
823 mode
== PIPE_RENDER_COND_BY_REGION_NO_WAIT
) {
824 perf_debug(&ice
->dbg
, "Conditional rendering demoted from "
825 "\"no wait\" to \"wait\".");
827 set_predicate_for_result(ice
, q
, condition
);
832 iris_resolve_conditional_render(struct iris_context
*ice
)
834 struct pipe_context
*ctx
= (void *) ice
;
835 struct iris_query
*q
= ice
->condition
.query
;
836 struct pipe_query
*query
= (void *) q
;
837 union pipe_query_result result
;
839 if (ice
->state
.predicate
!= IRIS_PREDICATE_STATE_USE_BIT
)
844 iris_get_query_result(ctx
, query
, true, &result
);
845 set_predicate_enable(ice
, (q
->result
!= 0) ^ ice
->condition
.condition
);
849 genX(init_query
)(struct iris_context
*ice
)
851 struct pipe_context
*ctx
= &ice
->ctx
;
853 ctx
->create_query
= iris_create_query
;
854 ctx
->create_batch_query
= iris_create_batch_query
;
855 ctx
->destroy_query
= iris_destroy_query
;
856 ctx
->begin_query
= iris_begin_query
;
857 ctx
->end_query
= iris_end_query
;
858 ctx
->get_query_result
= iris_get_query_result
;
859 ctx
->get_query_result_resource
= iris_get_query_result_resource
;
860 ctx
->set_active_query_state
= iris_set_active_query_state
;
861 ctx
->render_condition
= iris_render_condition
;
863 ice
->vtbl
.resolve_conditional_render
= iris_resolve_conditional_render
;