2 * Copyright © 2019 Google LLC
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
27 * Initialize a command stream.
30 tu_cs_init(struct tu_cs
*cs
,
31 struct tu_device
*device
,
33 uint32_t initial_size
)
35 assert(mode
!= TU_CS_MODE_EXTERNAL
);
37 memset(cs
, 0, sizeof(*cs
));
41 cs
->next_bo_size
= initial_size
;
45 * Initialize a command stream as a wrapper to an external buffer.
48 tu_cs_init_external(struct tu_cs
*cs
, uint32_t *start
, uint32_t *end
)
50 memset(cs
, 0, sizeof(*cs
));
52 cs
->mode
= TU_CS_MODE_EXTERNAL
;
53 cs
->start
= cs
->reserved_end
= cs
->cur
= start
;
58 * Finish and release all resources owned by a command stream.
61 tu_cs_finish(struct tu_cs
*cs
)
63 for (uint32_t i
= 0; i
< cs
->bo_count
; ++i
) {
64 tu_bo_finish(cs
->device
, cs
->bos
[i
]);
73 * Get the offset of the command packets emitted since the last call to
77 tu_cs_get_offset(const struct tu_cs
*cs
)
80 return cs
->start
- (uint32_t *) cs
->bos
[cs
->bo_count
- 1]->map
;
84 * Get the size of the command packets emitted since the last call to
88 tu_cs_get_size(const struct tu_cs
*cs
)
90 return cs
->cur
- cs
->start
;
94 * Return true if there is no command packet emitted since the last call to
98 tu_cs_is_empty(const struct tu_cs
*cs
)
100 return tu_cs_get_size(cs
) == 0;
104 * Allocate and add a BO to a command stream. Following command packets will
105 * be emitted to the new BO.
108 tu_cs_add_bo(struct tu_cs
*cs
, uint32_t size
)
110 /* no BO for TU_CS_MODE_EXTERNAL */
111 assert(cs
->mode
!= TU_CS_MODE_EXTERNAL
);
113 /* no dangling command packet */
114 assert(tu_cs_is_empty(cs
));
116 /* grow cs->bos if needed */
117 if (cs
->bo_count
== cs
->bo_capacity
) {
118 uint32_t new_capacity
= MAX2(4, 2 * cs
->bo_capacity
);
119 struct tu_bo
**new_bos
=
120 realloc(cs
->bos
, new_capacity
* sizeof(struct tu_bo
*));
122 return VK_ERROR_OUT_OF_HOST_MEMORY
;
124 cs
->bo_capacity
= new_capacity
;
128 struct tu_bo
*new_bo
= malloc(sizeof(struct tu_bo
));
130 return VK_ERROR_OUT_OF_HOST_MEMORY
;
133 tu_bo_init_new(cs
->device
, new_bo
, size
* sizeof(uint32_t));
134 if (result
!= VK_SUCCESS
) {
139 result
= tu_bo_map(cs
->device
, new_bo
);
140 if (result
!= VK_SUCCESS
) {
141 tu_bo_finish(cs
->device
, new_bo
);
146 cs
->bos
[cs
->bo_count
++] = new_bo
;
148 cs
->start
= cs
->cur
= cs
->reserved_end
= (uint32_t *) new_bo
->map
;
149 cs
->end
= cs
->start
+ new_bo
->size
/ sizeof(uint32_t);
155 * Reserve an IB entry.
158 tu_cs_reserve_entry(struct tu_cs
*cs
)
160 /* entries are only for TU_CS_MODE_GROW */
161 assert(cs
->mode
== TU_CS_MODE_GROW
);
163 /* grow cs->entries if needed */
164 if (cs
->entry_count
== cs
->entry_capacity
) {
165 uint32_t new_capacity
= MAX2(4, cs
->entry_capacity
* 2);
166 struct tu_cs_entry
*new_entries
=
167 realloc(cs
->entries
, new_capacity
* sizeof(struct tu_cs_entry
));
169 return VK_ERROR_OUT_OF_HOST_MEMORY
;
171 cs
->entry_capacity
= new_capacity
;
172 cs
->entries
= new_entries
;
179 * Add an IB entry for the command packets emitted since the last call to this
183 tu_cs_add_entry(struct tu_cs
*cs
)
185 /* entries are only for TU_CS_MODE_GROW */
186 assert(cs
->mode
== TU_CS_MODE_GROW
);
188 /* disallow empty entry */
189 assert(!tu_cs_is_empty(cs
));
192 * because we disallow empty entry, tu_cs_add_bo and tu_cs_reserve_entry
193 * must both have been called
195 assert(cs
->bo_count
);
196 assert(cs
->entry_count
< cs
->entry_capacity
);
198 /* add an entry for [cs->start, cs->cur] */
199 cs
->entries
[cs
->entry_count
++] = (struct tu_cs_entry
) {
200 .bo
= cs
->bos
[cs
->bo_count
- 1],
201 .size
= tu_cs_get_size(cs
) * sizeof(uint32_t),
202 .offset
= tu_cs_get_offset(cs
) * sizeof(uint32_t),
209 * same behavior as tu_cs_emit_call but without the indirect
212 tu_cs_add_entries(struct tu_cs
*cs
, struct tu_cs
*target
)
216 assert(cs
->mode
== TU_CS_MODE_GROW
);
217 assert(target
->mode
== TU_CS_MODE_GROW
);
219 if (!tu_cs_is_empty(cs
))
222 for (unsigned i
= 0; i
< target
->entry_count
; i
++) {
223 result
= tu_cs_reserve_entry(cs
);
224 if (result
!= VK_SUCCESS
)
226 cs
->entries
[cs
->entry_count
++] = target
->entries
[i
];
233 * Begin (or continue) command packet emission. This does nothing but sanity
234 * checks currently. \a cs must not be in TU_CS_MODE_SUB_STREAM mode.
237 tu_cs_begin(struct tu_cs
*cs
)
239 assert(cs
->mode
!= TU_CS_MODE_SUB_STREAM
);
240 assert(tu_cs_is_empty(cs
));
244 * End command packet emission. This adds an IB entry when \a cs is in
245 * TU_CS_MODE_GROW mode.
248 tu_cs_end(struct tu_cs
*cs
)
250 assert(cs
->mode
!= TU_CS_MODE_SUB_STREAM
);
252 if (cs
->mode
== TU_CS_MODE_GROW
&& !tu_cs_is_empty(cs
))
257 * Begin command packet emission to a sub-stream. \a cs must be in
258 * TU_CS_MODE_SUB_STREAM mode.
260 * Return \a sub_cs which is in TU_CS_MODE_EXTERNAL mode. tu_cs_begin and
261 * tu_cs_reserve_space are implied and \a sub_cs is ready for command packet
265 tu_cs_begin_sub_stream(struct tu_cs
*cs
, uint32_t size
, struct tu_cs
*sub_cs
)
267 assert(cs
->mode
== TU_CS_MODE_SUB_STREAM
);
270 VkResult result
= tu_cs_reserve_space(cs
, size
);
271 if (result
!= VK_SUCCESS
)
274 tu_cs_init_external(sub_cs
, cs
->cur
, cs
->reserved_end
);
276 result
= tu_cs_reserve_space(sub_cs
, size
);
277 assert(result
== VK_SUCCESS
);
283 * Allocate count*size dwords, aligned to size dwords.
284 * \a cs must be in TU_CS_MODE_SUB_STREAM mode.
288 tu_cs_alloc(struct tu_cs
*cs
,
291 struct ts_cs_memory
*memory
)
293 assert(cs
->mode
== TU_CS_MODE_SUB_STREAM
);
294 assert(size
&& size
<= 1024);
299 /* TODO: smarter way to deal with alignment? */
301 VkResult result
= tu_cs_reserve_space(cs
, count
* size
+ (size
-1));
302 if (result
!= VK_SUCCESS
)
305 struct tu_bo
*bo
= cs
->bos
[cs
->bo_count
- 1];
306 size_t offset
= align(tu_cs_get_offset(cs
), size
);
308 memory
->map
= bo
->map
+ offset
* sizeof(uint32_t);
309 memory
->iova
= bo
->iova
+ offset
* sizeof(uint32_t);
311 cs
->start
= cs
->cur
= (uint32_t*) bo
->map
+ offset
+ count
* size
;
317 * End command packet emission to a sub-stream. \a sub_cs becomes invalid
320 * Return an IB entry for the sub-stream. The entry has the same lifetime as
324 tu_cs_end_sub_stream(struct tu_cs
*cs
, struct tu_cs
*sub_cs
)
326 assert(cs
->mode
== TU_CS_MODE_SUB_STREAM
);
327 assert(cs
->bo_count
);
328 assert(sub_cs
->start
== cs
->cur
&& sub_cs
->end
== cs
->reserved_end
);
329 tu_cs_sanity_check(sub_cs
);
333 cs
->cur
= sub_cs
->cur
;
335 struct tu_cs_entry entry
= {
336 .bo
= cs
->bos
[cs
->bo_count
- 1],
337 .size
= tu_cs_get_size(cs
) * sizeof(uint32_t),
338 .offset
= tu_cs_get_offset(cs
) * sizeof(uint32_t),
347 * Reserve space from a command stream for \a reserved_size uint32_t values.
348 * This never fails when \a cs has mode TU_CS_MODE_EXTERNAL.
351 tu_cs_reserve_space(struct tu_cs
*cs
, uint32_t reserved_size
)
353 if (tu_cs_get_space(cs
) < reserved_size
) {
354 if (cs
->mode
== TU_CS_MODE_EXTERNAL
) {
355 unreachable("cannot grow external buffer");
356 return VK_ERROR_OUT_OF_HOST_MEMORY
;
359 /* add an entry for the exiting command packets */
360 if (!tu_cs_is_empty(cs
)) {
361 /* no direct command packet for TU_CS_MODE_SUB_STREAM */
362 assert(cs
->mode
!= TU_CS_MODE_SUB_STREAM
);
367 if (cs
->cond_flags
) {
368 /* Subtract one here to account for the DWORD field itself. */
369 *cs
->cond_dwords
= cs
->cur
- cs
->cond_dwords
- 1;
371 /* space for CP_COND_REG_EXEC in next bo */
375 /* switch to a new BO */
376 uint32_t new_size
= MAX2(cs
->next_bo_size
, reserved_size
);
377 VkResult result
= tu_cs_add_bo(cs
, new_size
);
378 if (result
!= VK_SUCCESS
)
381 /* if inside a condition, emit a new CP_COND_REG_EXEC */
382 if (cs
->cond_flags
) {
383 cs
->reserved_end
= cs
->cur
+ reserved_size
;
385 tu_cs_emit_pkt7(cs
, CP_COND_REG_EXEC
, 2);
386 tu_cs_emit(cs
, cs
->cond_flags
);
388 cs
->cond_dwords
= cs
->cur
;
390 /* Emit dummy DWORD field here */
391 tu_cs_emit(cs
, CP_COND_REG_EXEC_1_DWORDS(0));
394 /* double the size for the next bo */
396 if (cs
->next_bo_size
< new_size
)
397 cs
->next_bo_size
= new_size
;
400 assert(tu_cs_get_space(cs
) >= reserved_size
);
401 cs
->reserved_end
= cs
->cur
+ reserved_size
;
403 if (cs
->mode
== TU_CS_MODE_GROW
) {
404 /* reserve an entry for the next call to this function or tu_cs_end */
405 return tu_cs_reserve_entry(cs
);
412 * Reset a command stream to its initial state. This discards all comand
413 * packets in \a cs, but does not necessarily release all resources.
416 tu_cs_reset(struct tu_cs
*cs
)
418 if (cs
->mode
== TU_CS_MODE_EXTERNAL
) {
419 assert(!cs
->bo_count
&& !cs
->entry_count
);
420 cs
->reserved_end
= cs
->cur
= cs
->start
;
424 for (uint32_t i
= 0; i
+ 1 < cs
->bo_count
; ++i
) {
425 tu_bo_finish(cs
->device
, cs
->bos
[i
]);
430 cs
->bos
[0] = cs
->bos
[cs
->bo_count
- 1];
433 cs
->start
= cs
->cur
= cs
->reserved_end
= (uint32_t *) cs
->bos
[0]->map
;
434 cs
->end
= cs
->start
+ cs
->bos
[0]->size
/ sizeof(uint32_t);