2 * Copyright © 2017 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
26 * Batchbuffer and command submission module.
28 * Every API draw call results in a number of GPU commands, which we
29 * collect into a "batch buffer". Typically, many draw calls are grouped
30 * into a single batch to amortize command submission overhead.
32 * We submit batches to the kernel using the I915_GEM_EXECBUFFER2 ioctl.
33 * One critical piece of data is the "validation list", which contains a
34 * list of the buffer objects (BOs) which the commands in the GPU need.
35 * The kernel will make sure these are resident and pinned at the correct
36 * virtual memory address before executing our batch. If a BO is not in
37 * the validation list, it effectively does not exist, so take care.
40 #include "iris_batch.h"
41 #include "iris_bufmgr.h"
42 #include "iris_context.h"
43 #include "iris_fence.h"
45 #include "drm-uapi/i915_drm.h"
47 #include "util/hash_table.h"
49 #include "main/macros.h"
62 #define FILE_DEBUG_FLAG DEBUG_BUFMGR
64 /* Terminating the batch takes either 4 bytes for MI_BATCH_BUFFER_END
65 * or 12 bytes for MI_BATCH_BUFFER_START (when chaining). Plus, we may
66 * need an extra 4 bytes to pad out to the nearest QWord. So reserve 16.
68 #define BATCH_RESERVED 16
71 iris_batch_reset(struct iris_batch
*batch
);
74 num_fences(struct iris_batch
*batch
)
76 return util_dynarray_num_elements(&batch
->exec_fences
,
77 struct drm_i915_gem_exec_fence
);
81 * Debugging code to dump the fence list, used by INTEL_DEBUG=submit.
84 dump_fence_list(struct iris_batch
*batch
)
86 fprintf(stderr
, "Fence list (length %u): ", num_fences(batch
));
88 util_dynarray_foreach(&batch
->exec_fences
,
89 struct drm_i915_gem_exec_fence
, f
) {
90 fprintf(stderr
, "%s%u%s ",
91 (f
->flags
& I915_EXEC_FENCE_WAIT
) ? "..." : "",
93 (f
->flags
& I915_EXEC_FENCE_SIGNAL
) ? "!" : "");
96 fprintf(stderr
, "\n");
100 * Debugging code to dump the validation list, used by INTEL_DEBUG=submit.
103 dump_validation_list(struct iris_batch
*batch
)
105 fprintf(stderr
, "Validation list (length %d):\n", batch
->exec_count
);
107 for (int i
= 0; i
< batch
->exec_count
; i
++) {
108 uint64_t flags
= batch
->validation_list
[i
].flags
;
109 assert(batch
->validation_list
[i
].handle
==
110 batch
->exec_bos
[i
]->gem_handle
);
111 fprintf(stderr
, "[%2d]: %2d %-14s @ 0x%016llx (%"PRIu64
"B)\t %2d refs %s\n",
113 batch
->validation_list
[i
].handle
,
114 batch
->exec_bos
[i
]->name
,
115 batch
->validation_list
[i
].offset
,
116 batch
->exec_bos
[i
]->size
,
117 batch
->exec_bos
[i
]->refcount
,
118 (flags
& EXEC_OBJECT_WRITE
) ? " (write)" : "");
123 * Return BO information to the batch decoder (for debugging).
125 static struct gen_batch_decode_bo
126 decode_get_bo(void *v_batch
, bool ppgtt
, uint64_t address
)
128 struct iris_batch
*batch
= v_batch
;
132 for (int i
= 0; i
< batch
->exec_count
; i
++) {
133 struct iris_bo
*bo
= batch
->exec_bos
[i
];
134 /* The decoder zeroes out the top 16 bits, so we need to as well */
135 uint64_t bo_address
= bo
->gtt_offset
& (~0ull >> 16);
137 if (address
>= bo_address
&& address
< bo_address
+ bo
->size
) {
138 return (struct gen_batch_decode_bo
) {
141 .map
= iris_bo_map(batch
->dbg
, bo
, MAP_READ
) +
142 (address
- bo_address
),
147 return (struct gen_batch_decode_bo
) { };
151 * Decode the current batch.
154 decode_batch(struct iris_batch
*batch
)
156 void *map
= iris_bo_map(batch
->dbg
, batch
->exec_bos
[0], MAP_READ
);
157 gen_print_batch(&batch
->decoder
, map
, batch
->primary_batch_size
,
158 batch
->exec_bos
[0]->gtt_offset
, false);
162 iris_init_batch(struct iris_batch
*batch
,
163 struct iris_screen
*screen
,
164 struct iris_vtable
*vtbl
,
165 struct pipe_debug_callback
*dbg
,
166 struct pipe_device_reset_callback
*reset
,
167 struct iris_batch
*all_batches
,
168 enum iris_batch_name name
,
172 batch
->screen
= screen
;
175 batch
->reset
= reset
;
178 /* engine should be one of I915_EXEC_RENDER, I915_EXEC_BLT, etc. */
179 assert((engine
& ~I915_EXEC_RING_MASK
) == 0);
180 assert(util_bitcount(engine
) == 1);
181 batch
->engine
= engine
;
183 batch
->hw_ctx_id
= iris_create_hw_context(screen
->bufmgr
);
184 assert(batch
->hw_ctx_id
);
186 iris_hw_context_set_priority(screen
->bufmgr
, batch
->hw_ctx_id
, priority
);
188 util_dynarray_init(&batch
->exec_fences
, ralloc_context(NULL
));
189 util_dynarray_init(&batch
->syncpts
, ralloc_context(NULL
));
191 batch
->exec_count
= 0;
192 batch
->exec_array_size
= 100;
194 malloc(batch
->exec_array_size
* sizeof(batch
->exec_bos
[0]));
195 batch
->validation_list
=
196 malloc(batch
->exec_array_size
* sizeof(batch
->validation_list
[0]));
198 batch
->cache
.render
= _mesa_hash_table_create(NULL
, _mesa_hash_pointer
,
199 _mesa_key_pointer_equal
);
200 batch
->cache
.depth
= _mesa_set_create(NULL
, _mesa_hash_pointer
,
201 _mesa_key_pointer_equal
);
203 memset(batch
->other_batches
, 0, sizeof(batch
->other_batches
));
205 for (int i
= 0, j
= 0; i
< IRIS_BATCH_COUNT
; i
++) {
206 if (&all_batches
[i
] != batch
)
207 batch
->other_batches
[j
++] = &all_batches
[i
];
210 if (unlikely(INTEL_DEBUG
)) {
211 const unsigned decode_flags
=
212 GEN_BATCH_DECODE_FULL
|
213 ((INTEL_DEBUG
& DEBUG_COLOR
) ? GEN_BATCH_DECODE_IN_COLOR
: 0) |
214 GEN_BATCH_DECODE_OFFSETS
|
215 GEN_BATCH_DECODE_FLOATS
;
217 /* TODO: track state size so we can print the right # of entries */
218 gen_batch_decode_ctx_init(&batch
->decoder
, &screen
->devinfo
,
219 stderr
, decode_flags
, NULL
,
220 decode_get_bo
, NULL
, batch
);
221 batch
->decoder
.max_vbo_decoded_lines
= 32;
224 iris_batch_reset(batch
);
227 static struct drm_i915_gem_exec_object2
*
228 find_validation_entry(struct iris_batch
*batch
, struct iris_bo
*bo
)
230 unsigned index
= READ_ONCE(bo
->index
);
232 if (index
< batch
->exec_count
&& batch
->exec_bos
[index
] == bo
)
233 return &batch
->validation_list
[index
];
235 /* May have been shared between multiple active batches */
236 for (index
= 0; index
< batch
->exec_count
; index
++) {
237 if (batch
->exec_bos
[index
] == bo
)
238 return &batch
->validation_list
[index
];
245 * Add a buffer to the current batch's validation list.
247 * You must call this on any BO you wish to use in this batch, to ensure
248 * that it's resident when the GPU commands execute.
251 iris_use_pinned_bo(struct iris_batch
*batch
,
255 assert(bo
->kflags
& EXEC_OBJECT_PINNED
);
257 /* Never mark the workaround BO with EXEC_OBJECT_WRITE. We don't care
258 * about the order of any writes to that buffer, and marking it writable
259 * would introduce data dependencies between multiple batches which share
262 if (bo
== batch
->screen
->workaround_bo
)
265 struct drm_i915_gem_exec_object2
*existing_entry
=
266 find_validation_entry(batch
, bo
);
268 if (existing_entry
) {
269 /* The BO is already in the validation list; mark it writable */
271 existing_entry
->flags
|= EXEC_OBJECT_WRITE
;
276 if (bo
!= batch
->bo
) {
277 /* This is the first time our batch has seen this BO. Before we use it,
278 * we may need to flush and synchronize with other batches.
280 for (int b
= 0; b
< ARRAY_SIZE(batch
->other_batches
); b
++) {
281 struct drm_i915_gem_exec_object2
*other_entry
=
282 find_validation_entry(batch
->other_batches
[b
], bo
);
284 /* If the buffer is referenced by another batch, and either batch
285 * intends to write it, then flush the other batch and synchronize.
287 * Consider these cases:
289 * 1. They read, we read => No synchronization required.
290 * 2. They read, we write => Synchronize (they need the old value)
291 * 3. They write, we read => Synchronize (we need their new value)
292 * 4. They write, we write => Synchronize (order writes)
294 * The read/read case is very common, as multiple batches usually
295 * share a streaming state buffer or shader assembly buffer, and
296 * we want to avoid synchronizing in this case.
299 ((other_entry
->flags
& EXEC_OBJECT_WRITE
) || writable
)) {
300 iris_batch_flush(batch
->other_batches
[b
]);
301 iris_batch_add_syncpt(batch
, batch
->other_batches
[b
]->last_syncpt
,
302 I915_EXEC_FENCE_WAIT
);
307 /* Now, take a reference and add it to the validation list. */
308 iris_bo_reference(bo
);
310 if (batch
->exec_count
== batch
->exec_array_size
) {
311 batch
->exec_array_size
*= 2;
313 realloc(batch
->exec_bos
,
314 batch
->exec_array_size
* sizeof(batch
->exec_bos
[0]));
315 batch
->validation_list
=
316 realloc(batch
->validation_list
,
317 batch
->exec_array_size
* sizeof(batch
->validation_list
[0]));
320 batch
->validation_list
[batch
->exec_count
] =
321 (struct drm_i915_gem_exec_object2
) {
322 .handle
= bo
->gem_handle
,
323 .offset
= bo
->gtt_offset
,
324 .flags
= bo
->kflags
| (writable
? EXEC_OBJECT_WRITE
: 0),
327 bo
->index
= batch
->exec_count
;
328 batch
->exec_bos
[batch
->exec_count
] = bo
;
329 batch
->aperture_space
+= bo
->size
;
335 create_batch(struct iris_batch
*batch
)
337 struct iris_screen
*screen
= batch
->screen
;
338 struct iris_bufmgr
*bufmgr
= screen
->bufmgr
;
340 batch
->bo
= iris_bo_alloc(bufmgr
, "command buffer",
341 BATCH_SZ
+ BATCH_RESERVED
, IRIS_MEMZONE_OTHER
);
342 batch
->bo
->kflags
|= EXEC_OBJECT_CAPTURE
;
343 batch
->map
= iris_bo_map(NULL
, batch
->bo
, MAP_READ
| MAP_WRITE
);
344 batch
->map_next
= batch
->map
;
346 iris_use_pinned_bo(batch
, batch
->bo
, false);
350 iris_batch_reset(struct iris_batch
*batch
)
352 struct iris_screen
*screen
= batch
->screen
;
354 iris_bo_unreference(batch
->bo
);
355 batch
->primary_batch_size
= 0;
356 batch
->contains_draw
= false;
359 assert(batch
->bo
->index
== 0);
361 struct iris_syncpt
*syncpt
= iris_create_syncpt(screen
);
362 iris_batch_add_syncpt(batch
, syncpt
, I915_EXEC_FENCE_SIGNAL
);
363 iris_syncpt_reference(screen
, &syncpt
, NULL
);
365 iris_cache_sets_clear(batch
);
369 iris_batch_free(struct iris_batch
*batch
)
371 struct iris_screen
*screen
= batch
->screen
;
372 struct iris_bufmgr
*bufmgr
= screen
->bufmgr
;
374 for (int i
= 0; i
< batch
->exec_count
; i
++) {
375 iris_bo_unreference(batch
->exec_bos
[i
]);
377 free(batch
->exec_bos
);
378 free(batch
->validation_list
);
380 ralloc_free(batch
->exec_fences
.mem_ctx
);
382 util_dynarray_foreach(&batch
->syncpts
, struct iris_syncpt
*, s
)
383 iris_syncpt_reference(screen
, s
, NULL
);
384 ralloc_free(batch
->syncpts
.mem_ctx
);
386 iris_syncpt_reference(screen
, &batch
->last_syncpt
, NULL
);
388 iris_bo_unreference(batch
->bo
);
391 batch
->map_next
= NULL
;
393 iris_destroy_hw_context(bufmgr
, batch
->hw_ctx_id
);
395 _mesa_hash_table_destroy(batch
->cache
.render
, NULL
);
396 _mesa_set_destroy(batch
->cache
.depth
, NULL
);
398 if (unlikely(INTEL_DEBUG
))
399 gen_batch_decode_ctx_finish(&batch
->decoder
);
403 * If we've chained to a secondary batch, or are getting near to the end,
404 * then flush. This should only be called between draws.
407 iris_batch_maybe_flush(struct iris_batch
*batch
, unsigned estimate
)
409 if (batch
->bo
!= batch
->exec_bos
[0] ||
410 iris_batch_bytes_used(batch
) + estimate
>= BATCH_SZ
) {
411 iris_batch_flush(batch
);
416 iris_chain_to_new_batch(struct iris_batch
*batch
)
418 /* We only support chaining a single time. */
419 assert(batch
->bo
== batch
->exec_bos
[0]);
421 VG(void *map
= batch
->map
);
422 uint32_t *cmd
= batch
->map_next
;
423 uint64_t *addr
= batch
->map_next
+ 4;
424 batch
->map_next
+= 12;
426 /* No longer held by batch->bo, still held by validation list */
427 iris_bo_unreference(batch
->bo
);
428 batch
->primary_batch_size
= iris_batch_bytes_used(batch
);
431 /* Emit MI_BATCH_BUFFER_START to chain to another batch. */
432 *cmd
= (0x31 << 23) | (1 << 8) | (3 - 2);
433 *addr
= batch
->bo
->gtt_offset
;
435 VG(VALGRIND_CHECK_MEM_IS_DEFINED(map
, batch
->primary_batch_size
));
439 * Terminate a batch with MI_BATCH_BUFFER_END.
442 iris_finish_batch(struct iris_batch
*batch
)
444 /* Emit MI_BATCH_BUFFER_END to finish our batch. */
445 uint32_t *map
= batch
->map_next
;
447 map
[0] = (0xA << 23);
449 batch
->map_next
+= 4;
450 VG(VALGRIND_CHECK_MEM_IS_DEFINED(batch
->map
, iris_batch_bytes_used(batch
)));
452 if (batch
->bo
== batch
->exec_bos
[0])
453 batch
->primary_batch_size
= iris_batch_bytes_used(batch
);
457 * Replace our current GEM context with a new one (in case it got banned).
460 replace_hw_ctx(struct iris_batch
*batch
)
462 struct iris_screen
*screen
= batch
->screen
;
463 struct iris_bufmgr
*bufmgr
= screen
->bufmgr
;
465 uint32_t new_ctx
= iris_clone_hw_context(bufmgr
, batch
->hw_ctx_id
);
469 iris_destroy_hw_context(bufmgr
, batch
->hw_ctx_id
);
470 batch
->hw_ctx_id
= new_ctx
;
472 /* Notify the context that state must be re-initialized. */
473 iris_lost_context_state(batch
);
479 * Submit the batch to the GPU via execbuffer2.
482 submit_batch(struct iris_batch
*batch
)
484 iris_bo_unmap(batch
->bo
);
486 /* The requirement for using I915_EXEC_NO_RELOC are:
488 * The addresses written in the objects must match the corresponding
489 * reloc.gtt_offset which in turn must match the corresponding
492 * Any render targets written to in the batch must be flagged with
495 * To avoid stalling, execobject.offset should match the current
496 * address of that object within the active context.
498 struct drm_i915_gem_execbuffer2 execbuf
= {
499 .buffers_ptr
= (uintptr_t) batch
->validation_list
,
500 .buffer_count
= batch
->exec_count
,
501 .batch_start_offset
= 0,
502 /* This must be QWord aligned. */
503 .batch_len
= ALIGN(batch
->primary_batch_size
, 8),
504 .flags
= batch
->engine
|
506 I915_EXEC_BATCH_FIRST
|
507 I915_EXEC_HANDLE_LUT
,
508 .rsvd1
= batch
->hw_ctx_id
, /* rsvd1 is actually the context ID */
511 if (num_fences(batch
)) {
512 execbuf
.flags
|= I915_EXEC_FENCE_ARRAY
;
513 execbuf
.num_cliprects
= num_fences(batch
);
514 execbuf
.cliprects_ptr
=
515 (uintptr_t)util_dynarray_begin(&batch
->exec_fences
);
519 if (!batch
->screen
->no_hw
&&
520 drm_ioctl(batch
->screen
->fd
, DRM_IOCTL_I915_GEM_EXECBUFFER2
, &execbuf
))
523 for (int i
= 0; i
< batch
->exec_count
; i
++) {
524 struct iris_bo
*bo
= batch
->exec_bos
[i
];
529 iris_bo_unreference(bo
);
536 batch_name_to_string(enum iris_batch_name name
)
538 const char *names
[IRIS_BATCH_COUNT
] = {
539 [IRIS_BATCH_RENDER
] = "render",
540 [IRIS_BATCH_COMPUTE
] = "compute",
546 * Flush the batch buffer, submitting it to the GPU and resetting it so
547 * we're ready to emit the next batch.
549 * \param in_fence_fd is ignored if -1. Otherwise, this function takes
550 * ownership of the fd.
552 * \param out_fence_fd is ignored if NULL. Otherwise, the caller must
553 * take ownership of the returned fd.
556 _iris_batch_flush(struct iris_batch
*batch
, const char *file
, int line
)
558 struct iris_screen
*screen
= batch
->screen
;
560 if (iris_batch_bytes_used(batch
) == 0)
563 iris_finish_batch(batch
);
565 if (unlikely(INTEL_DEBUG
& (DEBUG_BATCH
| DEBUG_SUBMIT
))) {
566 int bytes_for_commands
= iris_batch_bytes_used(batch
);
567 int second_bytes
= 0;
568 if (batch
->bo
!= batch
->exec_bos
[0]) {
569 second_bytes
= bytes_for_commands
;
570 bytes_for_commands
+= batch
->primary_batch_size
;
572 fprintf(stderr
, "%19s:%-3d: %s batch [%u] flush with %5d+%5db (%0.1f%%) "
573 "(cmds), %4d BOs (%0.1fMb aperture)\n",
574 file
, line
, batch_name_to_string(batch
->name
), batch
->hw_ctx_id
,
575 batch
->primary_batch_size
, second_bytes
,
576 100.0f
* bytes_for_commands
/ BATCH_SZ
,
578 (float) batch
->aperture_space
/ (1024 * 1024));
579 dump_fence_list(batch
);
580 dump_validation_list(batch
);
583 if (unlikely(INTEL_DEBUG
& DEBUG_BATCH
)) {
587 int ret
= submit_batch(batch
);
589 batch
->exec_count
= 0;
590 batch
->aperture_space
= 0;
592 struct iris_syncpt
*syncpt
=
593 ((struct iris_syncpt
**) util_dynarray_begin(&batch
->syncpts
))[0];
594 iris_syncpt_reference(screen
, &batch
->last_syncpt
, syncpt
);
596 util_dynarray_foreach(&batch
->syncpts
, struct iris_syncpt
*, s
)
597 iris_syncpt_reference(screen
, s
, NULL
);
598 util_dynarray_clear(&batch
->syncpts
);
600 util_dynarray_clear(&batch
->exec_fences
);
602 if (unlikely(INTEL_DEBUG
& DEBUG_SYNC
)) {
603 dbg_printf("waiting for idle\n");
604 iris_bo_wait_rendering(batch
->bo
); /* if execbuf failed; this is a nop */
607 /* Start a new batch buffer. */
608 iris_batch_reset(batch
);
610 /* EIO means our context is banned. In this case, try and replace it
611 * with a new logical context, and inform iris_context that all state
612 * has been lost and needs to be re-initialized. If this succeeds,
613 * dubiously claim success...
615 if (ret
== -EIO
&& replace_hw_ctx(batch
)) {
616 if (batch
->reset
->reset
) {
617 /* Tell the state tracker the device is lost and it was our fault. */
618 batch
->reset
->reset(batch
->reset
->data
, PIPE_GUILTY_CONTEXT_RESET
);
625 //if (iris->ctx.Const.ResetStrategy == GL_LOSE_CONTEXT_ON_RESET_ARB)
626 //iris_check_for_reset(ice);
629 const bool color
= INTEL_DEBUG
& DEBUG_COLOR
;
630 fprintf(stderr
, "%siris: Failed to submit batchbuffer: %-80s%s\n",
631 color
? "\e[1;41m" : "", strerror(-ret
), color
? "\e[0m" : "");
638 * Does the current batch refer to the given BO?
640 * (In other words, is the BO in the current batch's validation list?)
643 iris_batch_references(struct iris_batch
*batch
, struct iris_bo
*bo
)
645 return find_validation_entry(batch
, bo
) != NULL
;