2 * Copyright © 2017 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
26 * Batchbuffer and command submission module.
28 * Every API draw call results in a number of GPU commands, which we
29 * collect into a "batch buffer". Typically, many draw calls are grouped
30 * into a single batch to amortize command submission overhead.
32 * We submit batches to the kernel using the I915_GEM_EXECBUFFER2 ioctl.
33 * One critical piece of data is the "validation list", which contains a
34 * list of the buffer objects (BOs) which the commands in the GPU need.
35 * The kernel will make sure these are resident and pinned at the correct
36 * virtual memory address before executing our batch. If a BO is not in
37 * the validation list, it effectively does not exist, so take care.
40 #include "iris_batch.h"
41 #include "iris_binder.h"
42 #include "iris_bufmgr.h"
43 #include "iris_context.h"
45 #include "drm-uapi/i915_drm.h"
47 #include "util/hash_table.h"
49 #include "main/macros.h"
54 #define FILE_DEBUG_FLAG DEBUG_BUFMGR
56 #define BATCH_SZ (20 * 1024)
58 /* Terminating the batch takes either 4 bytes for MI_BATCH_BUFFER_END
59 * or 12 bytes for MI_BATCH_BUFFER_START (when chaining). Plus, we may
60 * need an extra 4 bytes to pad out to the nearest QWord. So reserve 16.
62 #define BATCH_RESERVED 16
65 iris_batch_reset(struct iris_batch
*batch
);
68 * Debugging code to dump the validation list, used by INTEL_DEBUG=submit.
71 dump_validation_list(struct iris_batch
*batch
)
73 fprintf(stderr
, "Validation list (length %d):\n", batch
->exec_count
);
75 for (int i
= 0; i
< batch
->exec_count
; i
++) {
76 uint64_t flags
= batch
->validation_list
[i
].flags
;
77 assert(batch
->validation_list
[i
].handle
==
78 batch
->exec_bos
[i
]->gem_handle
);
79 fprintf(stderr
, "[%2d]: %2d %-14s %p %-7s @ 0x%016llx (%"PRIu64
"B) - %d refs\n",
81 batch
->validation_list
[i
].handle
,
82 batch
->exec_bos
[i
]->name
,
84 (flags
& EXEC_OBJECT_WRITE
) ? "(write)" : "",
85 batch
->validation_list
[i
].offset
,
86 batch
->exec_bos
[i
]->size
,
87 batch
->exec_bos
[i
]->refcount
);
92 * Return BO information to the batch decoder (for debugging).
94 static struct gen_batch_decode_bo
95 decode_get_bo(void *v_batch
, uint64_t address
)
97 struct iris_batch
*batch
= v_batch
;
99 for (int i
= 0; i
< batch
->exec_count
; i
++) {
100 struct iris_bo
*bo
= batch
->exec_bos
[i
];
101 /* The decoder zeroes out the top 16 bits, so we need to as well */
102 uint64_t bo_address
= bo
->gtt_offset
& (~0ull >> 16);
104 if (address
>= bo_address
&& address
< bo_address
+ bo
->size
) {
105 return (struct gen_batch_decode_bo
) {
108 .map
= iris_bo_map(batch
->dbg
, bo
, MAP_READ
) +
109 (address
- bo_address
),
114 return (struct gen_batch_decode_bo
) { };
118 * Decode the current batch.
121 decode_batch(struct iris_batch
*batch
)
123 void *map
= iris_bo_map(batch
->dbg
, batch
->exec_bos
[0], MAP_READ
);
124 gen_print_batch(&batch
->decoder
, map
, batch
->primary_batch_size
,
125 batch
->exec_bos
[0]->gtt_offset
);
129 uint_key_compare(const void *a
, const void *b
)
135 uint_key_hash(const void *key
)
137 return (uintptr_t) key
;
141 iris_init_batch(struct iris_batch
*batch
,
142 struct iris_screen
*screen
,
143 struct iris_vtable
*vtbl
,
144 struct pipe_debug_callback
*dbg
,
147 batch
->screen
= screen
;
151 /* engine should be one of I915_EXEC_RENDER, I915_EXEC_BLT, etc. */
152 assert((engine
& ~I915_EXEC_RING_MASK
) == 0);
153 assert(util_bitcount(engine
) == 1);
154 batch
->engine
= engine
;
156 batch
->exec_count
= 0;
157 batch
->exec_array_size
= 100;
159 malloc(batch
->exec_array_size
* sizeof(batch
->exec_bos
[0]));
160 batch
->validation_list
=
161 malloc(batch
->exec_array_size
* sizeof(batch
->validation_list
[0]));
163 batch
->binder
.bo
= NULL
;
165 batch
->cache
.render
= _mesa_hash_table_create(NULL
, _mesa_hash_pointer
,
166 _mesa_key_pointer_equal
);
167 batch
->cache
.depth
= _mesa_set_create(NULL
, _mesa_hash_pointer
,
168 _mesa_key_pointer_equal
);
169 if (unlikely(INTEL_DEBUG
)) {
171 _mesa_hash_table_create(NULL
, uint_key_hash
, uint_key_compare
);
173 const unsigned decode_flags
=
174 GEN_BATCH_DECODE_FULL
|
175 ((INTEL_DEBUG
& DEBUG_COLOR
) ? GEN_BATCH_DECODE_IN_COLOR
: 0) |
176 GEN_BATCH_DECODE_OFFSETS
|
177 GEN_BATCH_DECODE_FLOATS
;
179 gen_batch_decode_ctx_init(&batch
->decoder
, &screen
->devinfo
,
180 stderr
, decode_flags
, NULL
,
181 decode_get_bo
, NULL
, batch
);
182 batch
->decoder
.max_vbo_decoded_lines
= 32;
185 iris_batch_reset(batch
);
188 #define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x))
191 add_exec_bo(struct iris_batch
*batch
, struct iris_bo
*bo
)
193 unsigned index
= READ_ONCE(bo
->index
);
195 if (index
< batch
->exec_count
&& batch
->exec_bos
[index
] == bo
)
198 /* May have been shared between multiple active batches */
199 for (index
= 0; index
< batch
->exec_count
; index
++) {
200 if (batch
->exec_bos
[index
] == bo
)
204 iris_bo_reference(bo
);
206 if (batch
->exec_count
== batch
->exec_array_size
) {
207 batch
->exec_array_size
*= 2;
209 realloc(batch
->exec_bos
,
210 batch
->exec_array_size
* sizeof(batch
->exec_bos
[0]));
211 batch
->validation_list
=
212 realloc(batch
->validation_list
,
213 batch
->exec_array_size
* sizeof(batch
->validation_list
[0]));
216 batch
->validation_list
[batch
->exec_count
] =
217 (struct drm_i915_gem_exec_object2
) {
218 .handle
= bo
->gem_handle
,
219 .offset
= bo
->gtt_offset
,
223 bo
->index
= batch
->exec_count
;
224 batch
->exec_bos
[batch
->exec_count
] = bo
;
225 batch
->aperture_space
+= bo
->size
;
227 return batch
->exec_count
++;
231 create_batch(struct iris_batch
*batch
)
233 struct iris_screen
*screen
= batch
->screen
;
234 struct iris_bufmgr
*bufmgr
= screen
->bufmgr
;
236 batch
->bo
= iris_bo_alloc(bufmgr
, "command buffer",
237 BATCH_SZ
+ BATCH_RESERVED
, IRIS_MEMZONE_OTHER
);
238 batch
->bo
->kflags
|= EXEC_OBJECT_CAPTURE
;
239 batch
->map
= iris_bo_map(NULL
, batch
->bo
, MAP_READ
| MAP_WRITE
);
240 batch
->map_next
= batch
->map
;
241 batch
->contains_draw
= false;
243 add_exec_bo(batch
, batch
->bo
);
247 iris_batch_reset(struct iris_batch
*batch
)
249 if (batch
->last_bo
!= NULL
) {
250 iris_bo_unreference(batch
->last_bo
);
251 batch
->last_bo
= NULL
;
253 batch
->last_bo
= batch
->bo
;
254 batch
->primary_batch_size
= 0;
257 assert(batch
->bo
->index
== 0);
259 iris_destroy_binder(&batch
->binder
);
260 iris_init_binder(&batch
->binder
, batch
->bo
->bufmgr
);
262 if (batch
->state_sizes
)
263 _mesa_hash_table_clear(batch
->state_sizes
, NULL
);
265 iris_cache_sets_clear(batch
);
269 iris_batch_free(struct iris_batch
*batch
)
271 for (int i
= 0; i
< batch
->exec_count
; i
++) {
272 iris_bo_unreference(batch
->exec_bos
[i
]);
274 free(batch
->exec_bos
);
275 free(batch
->validation_list
);
276 iris_bo_unreference(batch
->bo
);
279 batch
->map_next
= NULL
;
281 iris_bo_unreference(batch
->last_bo
);
283 _mesa_hash_table_destroy(batch
->cache
.render
, NULL
);
284 _mesa_set_destroy(batch
->cache
.depth
, NULL
);
286 iris_destroy_binder(&batch
->binder
);
288 if (batch
->state_sizes
) {
289 _mesa_hash_table_destroy(batch
->state_sizes
, NULL
);
290 gen_batch_decode_ctx_finish(&batch
->decoder
);
295 batch_bytes_used(struct iris_batch
*batch
)
297 return batch
->map_next
- batch
->map
;
301 * If we've chained to a secondary batch, or are getting near to the end,
302 * then flush. This should only be called between draws.
305 iris_batch_maybe_flush(struct iris_batch
*batch
, unsigned estimate
)
307 if (batch
->bo
!= batch
->exec_bos
[0] ||
308 batch_bytes_used(batch
) + estimate
>= BATCH_SZ
) {
309 iris_batch_flush(batch
);
314 * Ensure the current command buffer has \param size bytes of space
315 * remaining. If not, this creates a secondary batch buffer and emits
316 * a jump from the primary batch to the start of the secondary.
318 * Most callers want iris_get_command_space() instead.
321 iris_require_command_space(struct iris_batch
*batch
, unsigned size
)
323 const unsigned required_bytes
= batch_bytes_used(batch
) + size
;
325 if (required_bytes
>= BATCH_SZ
) {
326 /* We only support chaining a single time. */
327 assert(batch
->bo
== batch
->exec_bos
[0]);
329 uint32_t *cmd
= batch
->map_next
;
330 uint64_t *addr
= batch
->map_next
+ 4;
331 batch
->map_next
+= 8;
333 /* No longer held by batch->bo, still held by validation list */
334 iris_bo_unreference(batch
->bo
);
335 batch
->primary_batch_size
= batch_bytes_used(batch
);
338 /* Emit MI_BATCH_BUFFER_START to chain to another batch. */
339 *cmd
= (0x31 << 23) | (1 << 8) | (3 - 2);
340 *addr
= batch
->bo
->gtt_offset
;
345 * Allocate space in the current command buffer, and return a pointer
346 * to the mapped area so the caller can write commands there.
348 * This should be called whenever emitting commands.
351 iris_get_command_space(struct iris_batch
*batch
, unsigned bytes
)
353 iris_require_command_space(batch
, bytes
);
354 void *map
= batch
->map_next
;
355 batch
->map_next
+= bytes
;
360 * Helper to emit GPU commands - allocates space, copies them there.
363 iris_batch_emit(struct iris_batch
*batch
, const void *data
, unsigned size
)
365 void *map
= iris_get_command_space(batch
, size
);
366 memcpy(map
, data
, size
);
370 * Terminate a batch with MI_BATCH_BUFFER_END.
373 iris_finish_batch(struct iris_batch
*batch
)
377 /* Emit MI_BATCH_BUFFER_END to finish our batch. */
378 uint32_t *map
= batch
->map_next
;
380 map
[0] = (0xA << 23);
382 batch
->map_next
+= 4;
384 if (batch
->bo
== batch
->exec_bos
[0])
385 batch
->primary_batch_size
= batch_bytes_used(batch
);
389 * Submit the batch to the GPU via execbuffer2.
392 submit_batch(struct iris_batch
*batch
, int in_fence_fd
, int *out_fence_fd
)
394 iris_bo_unmap(batch
->bo
);
396 /* The requirement for using I915_EXEC_NO_RELOC are:
398 * The addresses written in the objects must match the corresponding
399 * reloc.gtt_offset which in turn must match the corresponding
402 * Any render targets written to in the batch must be flagged with
405 * To avoid stalling, execobject.offset should match the current
406 * address of that object within the active context.
408 struct drm_i915_gem_execbuffer2 execbuf
= {
409 .buffers_ptr
= (uintptr_t) batch
->validation_list
,
410 .buffer_count
= batch
->exec_count
,
411 .batch_start_offset
= 0,
412 /* This must be QWord aligned. */
413 .batch_len
= ALIGN(batch
->primary_batch_size
, 8),
414 .flags
= batch
->engine
|
416 I915_EXEC_BATCH_FIRST
|
417 I915_EXEC_HANDLE_LUT
,
418 .rsvd1
= batch
->hw_ctx_id
, /* rsvd1 is actually the context ID */
421 unsigned long cmd
= DRM_IOCTL_I915_GEM_EXECBUFFER2
;
423 if (in_fence_fd
!= -1) {
424 execbuf
.rsvd2
= in_fence_fd
;
425 execbuf
.flags
|= I915_EXEC_FENCE_IN
;
428 if (out_fence_fd
!= NULL
) {
429 cmd
= DRM_IOCTL_I915_GEM_EXECBUFFER2_WR
;
431 execbuf
.flags
|= I915_EXEC_FENCE_OUT
;
434 int ret
= drm_ioctl(batch
->screen
->fd
, cmd
, &execbuf
);
437 DBG("execbuf FAILED: errno = %d\n", -ret
);
438 fprintf(stderr
, "execbuf FAILED: errno = %d\n", -ret
);
441 DBG("execbuf succeeded\n");
444 for (int i
= 0; i
< batch
->exec_count
; i
++) {
445 struct iris_bo
*bo
= batch
->exec_bos
[i
];
451 if (ret
== 0 && out_fence_fd
!= NULL
)
452 *out_fence_fd
= execbuf
.rsvd2
>> 32;
458 * Flush the batch buffer, submitting it to the GPU and resetting it so
459 * we're ready to emit the next batch.
461 * \param in_fence_fd is ignored if -1. Otherwise, this function takes
462 * ownership of the fd.
464 * \param out_fence_fd is ignored if NULL. Otherwise, the caller must
465 * take ownership of the returned fd.
468 _iris_batch_flush_fence(struct iris_batch
*batch
,
469 int in_fence_fd
, int *out_fence_fd
,
470 const char *file
, int line
)
472 if (batch_bytes_used(batch
) == 0)
475 iris_finish_batch(batch
);
477 if (unlikely(INTEL_DEBUG
& (DEBUG_BATCH
| DEBUG_SUBMIT
))) {
478 int bytes_for_commands
= batch_bytes_used(batch
);
479 int bytes_for_binder
= batch
->binder
.insert_point
;
480 int second_bytes
= 0;
481 if (batch
->bo
!= batch
->exec_bos
[0]) {
482 second_bytes
= bytes_for_commands
;
483 bytes_for_commands
+= batch
->primary_batch_size
;
485 fprintf(stderr
, "%19s:%-3d: Batchbuffer flush with %5d+%5db (%0.1f%%) "
486 "(cmds), %5db (%0.1f%%) (binder), %4d BOs (%0.1fMb aperture)\n",
488 batch
->primary_batch_size
, second_bytes
,
489 100.0f
* bytes_for_commands
/ BATCH_SZ
,
490 bytes_for_binder
, 100.0f
* bytes_for_binder
/ IRIS_BINDER_SIZE
,
492 (float) batch
->aperture_space
/ (1024 * 1024));
493 dump_validation_list(batch
);
496 if (unlikely(INTEL_DEBUG
& DEBUG_BATCH
)) {
500 int ret
= submit_batch(batch
, in_fence_fd
, out_fence_fd
);
505 //if (iris->ctx.Const.ResetStrategy == GL_LOSE_CONTEXT_ON_RESET_ARB)
506 //iris_check_for_reset(ice);
508 if (unlikely(INTEL_DEBUG
& DEBUG_SYNC
)) {
509 dbg_printf("waiting for idle\n");
510 iris_bo_wait_rendering(batch
->bo
);
514 const bool color
= INTEL_DEBUG
& DEBUG_COLOR
;
515 fprintf(stderr
, "%siris: Failed to submit batchbuffer: %-80s%s\n",
516 color
? "\e[1;41m" : "", strerror(-ret
), color
? "\e[0m" : "");
521 /* Clean up after the batch we submitted and prepare for a new one. */
522 for (int i
= 0; i
< batch
->exec_count
; i
++) {
523 iris_bo_unreference(batch
->exec_bos
[i
]);
524 batch
->exec_bos
[i
] = NULL
;
526 batch
->exec_count
= 0;
527 batch
->aperture_space
= 0;
529 /* Start a new batch buffer. */
530 iris_batch_reset(batch
);
536 * Does the current batch refer to the given BO?
538 * (In other words, is the BO in the current batch's validation list?)
541 iris_batch_references(struct iris_batch
*batch
, struct iris_bo
*bo
)
543 unsigned index
= READ_ONCE(bo
->index
);
544 if (index
< batch
->exec_count
&& batch
->exec_bos
[index
] == bo
)
547 for (int i
= 0; i
< batch
->exec_count
; i
++) {
548 if (batch
->exec_bos
[i
] == bo
)
555 * Add a buffer to the current batch's validation list.
557 * You must call this on any BO you wish to use in this batch, to ensure
558 * that it's resident when the GPU commands execute.
561 iris_use_pinned_bo(struct iris_batch
*batch
,
565 assert(bo
->kflags
& EXEC_OBJECT_PINNED
);
566 unsigned index
= add_exec_bo(batch
, bo
);
568 batch
->validation_list
[index
].flags
|= EXEC_OBJECT_WRITE
;