2 * Copyright © 2019 Google LLC
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
27 * Initialize a command stream.
30 tu_cs_init(struct tu_cs
*cs
,
31 struct tu_device
*device
,
33 uint32_t initial_size
)
35 assert(mode
!= TU_CS_MODE_EXTERNAL
);
37 memset(cs
, 0, sizeof(*cs
));
41 cs
->next_bo_size
= initial_size
;
45 * Initialize a command stream as a wrapper to an external buffer.
48 tu_cs_init_external(struct tu_cs
*cs
, uint32_t *start
, uint32_t *end
)
50 memset(cs
, 0, sizeof(*cs
));
52 cs
->mode
= TU_CS_MODE_EXTERNAL
;
53 cs
->start
= cs
->reserved_end
= cs
->cur
= start
;
58 * Finish and release all resources owned by a command stream.
61 tu_cs_finish(struct tu_cs
*cs
)
63 for (uint32_t i
= 0; i
< cs
->bo_count
; ++i
) {
64 tu_bo_finish(cs
->device
, cs
->bos
[i
]);
73 * Get the offset of the command packets emitted since the last call to
77 tu_cs_get_offset(const struct tu_cs
*cs
)
80 return cs
->start
- (uint32_t *) cs
->bos
[cs
->bo_count
- 1]->map
;
84 * Allocate and add a BO to a command stream. Following command packets will
85 * be emitted to the new BO.
88 tu_cs_add_bo(struct tu_cs
*cs
, uint32_t size
)
90 /* no BO for TU_CS_MODE_EXTERNAL */
91 assert(cs
->mode
!= TU_CS_MODE_EXTERNAL
);
93 /* no dangling command packet */
94 assert(tu_cs_is_empty(cs
));
96 /* grow cs->bos if needed */
97 if (cs
->bo_count
== cs
->bo_capacity
) {
98 uint32_t new_capacity
= MAX2(4, 2 * cs
->bo_capacity
);
99 struct tu_bo
**new_bos
=
100 realloc(cs
->bos
, new_capacity
* sizeof(struct tu_bo
*));
102 return VK_ERROR_OUT_OF_HOST_MEMORY
;
104 cs
->bo_capacity
= new_capacity
;
108 struct tu_bo
*new_bo
= malloc(sizeof(struct tu_bo
));
110 return VK_ERROR_OUT_OF_HOST_MEMORY
;
113 tu_bo_init_new(cs
->device
, new_bo
, size
* sizeof(uint32_t));
114 if (result
!= VK_SUCCESS
) {
119 result
= tu_bo_map(cs
->device
, new_bo
);
120 if (result
!= VK_SUCCESS
) {
121 tu_bo_finish(cs
->device
, new_bo
);
126 cs
->bos
[cs
->bo_count
++] = new_bo
;
128 cs
->start
= cs
->cur
= cs
->reserved_end
= (uint32_t *) new_bo
->map
;
129 cs
->end
= cs
->start
+ new_bo
->size
/ sizeof(uint32_t);
135 * Reserve an IB entry.
138 tu_cs_reserve_entry(struct tu_cs
*cs
)
140 /* entries are only for TU_CS_MODE_GROW */
141 assert(cs
->mode
== TU_CS_MODE_GROW
);
143 /* grow cs->entries if needed */
144 if (cs
->entry_count
== cs
->entry_capacity
) {
145 uint32_t new_capacity
= MAX2(4, cs
->entry_capacity
* 2);
146 struct tu_cs_entry
*new_entries
=
147 realloc(cs
->entries
, new_capacity
* sizeof(struct tu_cs_entry
));
149 return VK_ERROR_OUT_OF_HOST_MEMORY
;
151 cs
->entry_capacity
= new_capacity
;
152 cs
->entries
= new_entries
;
159 * Add an IB entry for the command packets emitted since the last call to this
163 tu_cs_add_entry(struct tu_cs
*cs
)
165 /* entries are only for TU_CS_MODE_GROW */
166 assert(cs
->mode
== TU_CS_MODE_GROW
);
168 /* disallow empty entry */
169 assert(!tu_cs_is_empty(cs
));
172 * because we disallow empty entry, tu_cs_add_bo and tu_cs_reserve_entry
173 * must both have been called
175 assert(cs
->bo_count
);
176 assert(cs
->entry_count
< cs
->entry_capacity
);
178 /* add an entry for [cs->start, cs->cur] */
179 cs
->entries
[cs
->entry_count
++] = (struct tu_cs_entry
) {
180 .bo
= cs
->bos
[cs
->bo_count
- 1],
181 .size
= tu_cs_get_size(cs
) * sizeof(uint32_t),
182 .offset
= tu_cs_get_offset(cs
) * sizeof(uint32_t),
189 * same behavior as tu_cs_emit_call but without the indirect
192 tu_cs_add_entries(struct tu_cs
*cs
, struct tu_cs
*target
)
196 assert(cs
->mode
== TU_CS_MODE_GROW
);
197 assert(target
->mode
== TU_CS_MODE_GROW
);
199 if (!tu_cs_is_empty(cs
))
202 for (unsigned i
= 0; i
< target
->entry_count
; i
++) {
203 result
= tu_cs_reserve_entry(cs
);
204 if (result
!= VK_SUCCESS
)
206 cs
->entries
[cs
->entry_count
++] = target
->entries
[i
];
213 * Begin (or continue) command packet emission. This does nothing but sanity
214 * checks currently. \a cs must not be in TU_CS_MODE_SUB_STREAM mode.
217 tu_cs_begin(struct tu_cs
*cs
)
219 assert(cs
->mode
!= TU_CS_MODE_SUB_STREAM
);
220 assert(tu_cs_is_empty(cs
));
224 * End command packet emission. This adds an IB entry when \a cs is in
225 * TU_CS_MODE_GROW mode.
228 tu_cs_end(struct tu_cs
*cs
)
230 assert(cs
->mode
!= TU_CS_MODE_SUB_STREAM
);
232 if (cs
->mode
== TU_CS_MODE_GROW
&& !tu_cs_is_empty(cs
))
237 * Begin command packet emission to a sub-stream. \a cs must be in
238 * TU_CS_MODE_SUB_STREAM mode.
240 * Return \a sub_cs which is in TU_CS_MODE_EXTERNAL mode. tu_cs_begin and
241 * tu_cs_reserve_space are implied and \a sub_cs is ready for command packet
245 tu_cs_begin_sub_stream(struct tu_cs
*cs
, uint32_t size
, struct tu_cs
*sub_cs
)
247 assert(cs
->mode
== TU_CS_MODE_SUB_STREAM
);
250 VkResult result
= tu_cs_reserve_space(cs
, size
);
251 if (result
!= VK_SUCCESS
)
254 tu_cs_init_external(sub_cs
, cs
->cur
, cs
->reserved_end
);
256 result
= tu_cs_reserve_space(sub_cs
, size
);
257 assert(result
== VK_SUCCESS
);
263 * Allocate count*size dwords, aligned to size dwords.
264 * \a cs must be in TU_CS_MODE_SUB_STREAM mode.
268 tu_cs_alloc(struct tu_cs
*cs
,
271 struct tu_cs_memory
*memory
)
273 assert(cs
->mode
== TU_CS_MODE_SUB_STREAM
);
274 assert(size
&& size
<= 1024);
279 /* TODO: smarter way to deal with alignment? */
281 VkResult result
= tu_cs_reserve_space(cs
, count
* size
+ (size
-1));
282 if (result
!= VK_SUCCESS
)
285 struct tu_bo
*bo
= cs
->bos
[cs
->bo_count
- 1];
286 size_t offset
= align(tu_cs_get_offset(cs
), size
);
288 memory
->map
= bo
->map
+ offset
* sizeof(uint32_t);
289 memory
->iova
= bo
->iova
+ offset
* sizeof(uint32_t);
291 cs
->start
= cs
->cur
= (uint32_t*) bo
->map
+ offset
+ count
* size
;
297 * End command packet emission to a sub-stream. \a sub_cs becomes invalid
300 * Return an IB entry for the sub-stream. The entry has the same lifetime as
304 tu_cs_end_sub_stream(struct tu_cs
*cs
, struct tu_cs
*sub_cs
)
306 assert(cs
->mode
== TU_CS_MODE_SUB_STREAM
);
307 assert(cs
->bo_count
);
308 assert(sub_cs
->start
== cs
->cur
&& sub_cs
->end
== cs
->reserved_end
);
309 tu_cs_sanity_check(sub_cs
);
313 cs
->cur
= sub_cs
->cur
;
315 struct tu_cs_entry entry
= {
316 .bo
= cs
->bos
[cs
->bo_count
- 1],
317 .size
= tu_cs_get_size(cs
) * sizeof(uint32_t),
318 .offset
= tu_cs_get_offset(cs
) * sizeof(uint32_t),
327 * Reserve space from a command stream for \a reserved_size uint32_t values.
328 * This never fails when \a cs has mode TU_CS_MODE_EXTERNAL.
331 tu_cs_reserve_space(struct tu_cs
*cs
, uint32_t reserved_size
)
333 if (tu_cs_get_space(cs
) < reserved_size
) {
334 if (cs
->mode
== TU_CS_MODE_EXTERNAL
) {
335 unreachable("cannot grow external buffer");
336 return VK_ERROR_OUT_OF_HOST_MEMORY
;
339 /* add an entry for the exiting command packets */
340 if (!tu_cs_is_empty(cs
)) {
341 /* no direct command packet for TU_CS_MODE_SUB_STREAM */
342 assert(cs
->mode
!= TU_CS_MODE_SUB_STREAM
);
347 if (cs
->cond_flags
) {
348 /* Subtract one here to account for the DWORD field itself. */
349 *cs
->cond_dwords
= cs
->cur
- cs
->cond_dwords
- 1;
351 /* space for CP_COND_REG_EXEC in next bo */
355 /* switch to a new BO */
356 uint32_t new_size
= MAX2(cs
->next_bo_size
, reserved_size
);
357 VkResult result
= tu_cs_add_bo(cs
, new_size
);
358 if (result
!= VK_SUCCESS
)
361 /* if inside a condition, emit a new CP_COND_REG_EXEC */
362 if (cs
->cond_flags
) {
363 cs
->reserved_end
= cs
->cur
+ reserved_size
;
365 tu_cs_emit_pkt7(cs
, CP_COND_REG_EXEC
, 2);
366 tu_cs_emit(cs
, cs
->cond_flags
);
368 cs
->cond_dwords
= cs
->cur
;
370 /* Emit dummy DWORD field here */
371 tu_cs_emit(cs
, CP_COND_REG_EXEC_1_DWORDS(0));
374 /* double the size for the next bo */
376 if (cs
->next_bo_size
< new_size
)
377 cs
->next_bo_size
= new_size
;
380 assert(tu_cs_get_space(cs
) >= reserved_size
);
381 cs
->reserved_end
= cs
->cur
+ reserved_size
;
383 if (cs
->mode
== TU_CS_MODE_GROW
) {
384 /* reserve an entry for the next call to this function or tu_cs_end */
385 return tu_cs_reserve_entry(cs
);
392 * Reset a command stream to its initial state. This discards all comand
393 * packets in \a cs, but does not necessarily release all resources.
396 tu_cs_reset(struct tu_cs
*cs
)
398 if (cs
->mode
== TU_CS_MODE_EXTERNAL
) {
399 assert(!cs
->bo_count
&& !cs
->entry_count
);
400 cs
->reserved_end
= cs
->cur
= cs
->start
;
404 for (uint32_t i
= 0; i
+ 1 < cs
->bo_count
; ++i
) {
405 tu_bo_finish(cs
->device
, cs
->bos
[i
]);
410 cs
->bos
[0] = cs
->bos
[cs
->bo_count
- 1];
413 cs
->start
= cs
->cur
= cs
->reserved_end
= (uint32_t *) cs
->bos
[0]->map
;
414 cs
->end
= cs
->start
+ cs
->bos
[0]->size
/ sizeof(uint32_t);