#include "tu_cs.h"
+/**
+ * Initialize a command stream.
+ */
void
-tu_cs_init(struct tu_cs *cs)
+tu_cs_init(struct tu_cs *cs,
+ struct tu_device *device,
+ enum tu_cs_mode mode,
+ uint32_t initial_size)
{
- cs->start = cs->cur = cs->end = NULL;
+ assert(mode != TU_CS_MODE_EXTERNAL);
- cs->entry_count = cs->entry_capacity = 0;
- cs->entries = NULL;
+ memset(cs, 0, sizeof(*cs));
- cs->bo_count = cs->bo_capacity = 0;
- cs->bos = NULL;
+ cs->device = device;
+ cs->mode = mode;
+ cs->next_bo_size = initial_size;
}
+/**
+ * Initialize a command stream as a wrapper to an external buffer.
+ */
void
-tu_cs_finish(struct tu_device *dev, struct tu_cs *cs)
+tu_cs_init_external(struct tu_cs *cs, uint32_t *start, uint32_t *end)
+{
+ memset(cs, 0, sizeof(*cs));
+
+ cs->mode = TU_CS_MODE_EXTERNAL;
+ cs->start = cs->reserved_end = cs->cur = start;
+ cs->end = end;
+}
+
+/**
+ * Finish and release all resources owned by a command stream.
+ */
+void
+tu_cs_finish(struct tu_cs *cs)
{
for (uint32_t i = 0; i < cs->bo_count; ++i) {
- tu_bo_finish(dev, cs->bos[i]);
+ tu_bo_finish(cs->device, cs->bos[i]);
free(cs->bos[i]);
}
free(cs->bos);
}
-VkResult
-tu_cs_begin(struct tu_device *dev, struct tu_cs *cs, uint32_t reserve_size)
+/**
+ * Get the offset of the command packets emitted since the last call to
+ * tu_cs_add_entry.
+ */
+static uint32_t
+tu_cs_get_offset(const struct tu_cs *cs)
{
- assert(reserve_size);
-
- if (cs->end - cs->cur < reserve_size) {
- if (cs->bo_count == cs->bo_capacity) {
- uint32_t new_capacity = MAX2(4, 2 * cs->bo_capacity);
- struct tu_bo **new_bos =
- realloc(cs->bos, new_capacity * sizeof(struct tu_bo *));
- if (!new_bos)
- abort();
-
- cs->bo_capacity = new_capacity;
- cs->bos = new_bos;
- }
+ assert(cs->bo_count);
+ return cs->start - (uint32_t *) cs->bos[cs->bo_count - 1]->map;
+}
- uint32_t new_size = MAX2(16384, reserve_size * sizeof(uint32_t));
- if (cs->bo_count)
- new_size = MAX2(new_size, cs->bos[cs->bo_count - 1]->size * 2);
+/*
+ * Allocate and add a BO to a command stream. Following command packets will
+ * be emitted to the new BO.
+ */
+static VkResult
+tu_cs_add_bo(struct tu_cs *cs, uint32_t size)
+{
+ /* no BO for TU_CS_MODE_EXTERNAL */
+ assert(cs->mode != TU_CS_MODE_EXTERNAL);
- struct tu_bo *new_bo = malloc(sizeof(struct tu_bo));
- if (!new_bo)
- abort();
+ /* no dangling command packet */
+ assert(tu_cs_is_empty(cs));
- VkResult result = tu_bo_init_new(dev, new_bo, new_size);
- if (result != VK_SUCCESS) {
- free(new_bo);
- return result;
- }
+ /* grow cs->bos if needed */
+ if (cs->bo_count == cs->bo_capacity) {
+ uint32_t new_capacity = MAX2(4, 2 * cs->bo_capacity);
+ struct tu_bo **new_bos =
+ realloc(cs->bos, new_capacity * sizeof(struct tu_bo *));
+ if (!new_bos)
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
- result = tu_bo_map(dev, new_bo);
- if (result != VK_SUCCESS) {
- tu_bo_finish(dev, new_bo);
- free(new_bo);
- return result;
- }
+ cs->bo_capacity = new_capacity;
+ cs->bos = new_bos;
+ }
- cs->bos[cs->bo_count] = new_bo;
- ++cs->bo_count;
+ struct tu_bo *new_bo = malloc(sizeof(struct tu_bo));
+ if (!new_bo)
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
- cs->start = cs->cur = (uint32_t *) new_bo->map;
- cs->end = cs->start + new_bo->size / sizeof(uint32_t);
+ VkResult result =
+ tu_bo_init_new(cs->device, new_bo, size * sizeof(uint32_t));
+ if (result != VK_SUCCESS) {
+ free(new_bo);
+ return result;
}
- cs->start = cs->cur;
+
+ result = tu_bo_map(cs->device, new_bo);
+ if (result != VK_SUCCESS) {
+ tu_bo_finish(cs->device, new_bo);
+ free(new_bo);
+ return result;
+ }
+
+ cs->bos[cs->bo_count++] = new_bo;
+
+ cs->start = cs->cur = cs->reserved_end = (uint32_t *) new_bo->map;
+ cs->end = cs->start + new_bo->size / sizeof(uint32_t);
return VK_SUCCESS;
}
-VkResult
-tu_cs_end(struct tu_cs *cs)
+/**
+ * Reserve an IB entry.
+ */
+static VkResult
+tu_cs_reserve_entry(struct tu_cs *cs)
{
- if (cs->start == cs->cur)
- return VK_SUCCESS;
+ /* entries are only for TU_CS_MODE_GROW */
+ assert(cs->mode == TU_CS_MODE_GROW);
- if (cs->entry_capacity == cs->entry_count) {
- uint32_t new_capacity = MAX2(cs->entry_capacity * 2, 4);
+ /* grow cs->entries if needed */
+ if (cs->entry_count == cs->entry_capacity) {
+ uint32_t new_capacity = MAX2(4, cs->entry_capacity * 2);
struct tu_cs_entry *new_entries =
realloc(cs->entries, new_capacity * sizeof(struct tu_cs_entry));
if (!new_entries)
- abort(); /* TODO */
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
- cs->entries = new_entries;
cs->entry_capacity = new_capacity;
+ cs->entries = new_entries;
}
+ return VK_SUCCESS;
+}
+
+/**
+ * Add an IB entry for the command packets emitted since the last call to this
+ * function.
+ */
+static void
+tu_cs_add_entry(struct tu_cs *cs)
+{
+ /* entries are only for TU_CS_MODE_GROW */
+ assert(cs->mode == TU_CS_MODE_GROW);
+
+ /* disallow empty entry */
+ assert(!tu_cs_is_empty(cs));
+
+ /*
+ * because we disallow empty entry, tu_cs_add_bo and tu_cs_reserve_entry
+ * must both have been called
+ */
assert(cs->bo_count);
+ assert(cs->entry_count < cs->entry_capacity);
+
+ /* add an entry for [cs->start, cs->cur] */
+ cs->entries[cs->entry_count++] = (struct tu_cs_entry) {
+ .bo = cs->bos[cs->bo_count - 1],
+ .size = tu_cs_get_size(cs) * sizeof(uint32_t),
+ .offset = tu_cs_get_offset(cs) * sizeof(uint32_t),
+ };
+
+ cs->start = cs->cur;
+}
- struct tu_cs_entry entry;
- entry.bo = cs->bos[cs->bo_count - 1];
- entry.size = (cs->cur - cs->start) * sizeof(uint32_t);
- entry.offset = (cs->start - (uint32_t *) entry.bo->map) * sizeof(uint32_t);
+/**
+ * same behavior as tu_cs_emit_call but without the indirect
+ */
+VkResult
+tu_cs_add_entries(struct tu_cs *cs, struct tu_cs *target)
+{
+ VkResult result;
- cs->entries[cs->entry_count] = entry;
- ++cs->entry_count;
+ assert(cs->mode == TU_CS_MODE_GROW);
+ assert(target->mode == TU_CS_MODE_GROW);
+
+ if (!tu_cs_is_empty(cs))
+ tu_cs_add_entry(cs);
+
+ for (unsigned i = 0; i < target->entry_count; i++) {
+ result = tu_cs_reserve_entry(cs);
+ if (result != VK_SUCCESS)
+ return result;
+ cs->entries[cs->entry_count++] = target->entries[i];
+ }
return VK_SUCCESS;
}
+/**
+ * Begin (or continue) command packet emission. This does nothing but sanity
+ * checks currently. \a cs must not be in TU_CS_MODE_SUB_STREAM mode.
+ */
void
-tu_cs_reset(struct tu_device *dev, struct tu_cs *cs)
+tu_cs_begin(struct tu_cs *cs)
{
- for (uint32_t i = 0; i + 1 < cs->bo_count; ++i) {
- tu_bo_finish(dev, cs->bos[i]);
- free(cs->bos[i]);
- }
+ assert(cs->mode != TU_CS_MODE_SUB_STREAM);
+ assert(tu_cs_is_empty(cs));
+}
- if (cs->bo_count) {
- cs->bos[0] = cs->bos[cs->bo_count - 1];
- cs->bo_count = 1;
+/**
+ * End command packet emission. This adds an IB entry when \a cs is in
+ * TU_CS_MODE_GROW mode.
+ */
+void
+tu_cs_end(struct tu_cs *cs)
+{
+ assert(cs->mode != TU_CS_MODE_SUB_STREAM);
- cs->start = cs->cur = (uint32_t *) cs->bos[0]->map;
- cs->end = cs->start + cs->bos[0]->size / sizeof(uint32_t);
- }
+ if (cs->mode == TU_CS_MODE_GROW && !tu_cs_is_empty(cs))
+ tu_cs_add_entry(cs);
+}
- cs->entry_count = 0;
+/**
+ * Begin command packet emission to a sub-stream. \a cs must be in
+ * TU_CS_MODE_SUB_STREAM mode.
+ *
+ * Return \a sub_cs which is in TU_CS_MODE_EXTERNAL mode. tu_cs_begin and
+ * tu_cs_reserve_space are implied and \a sub_cs is ready for command packet
+ * emission.
+ */
+VkResult
+tu_cs_begin_sub_stream(struct tu_cs *cs, uint32_t size, struct tu_cs *sub_cs)
+{
+ assert(cs->mode == TU_CS_MODE_SUB_STREAM);
+ assert(size);
+
+ VkResult result = tu_cs_reserve_space(cs, size);
+ if (result != VK_SUCCESS)
+ return result;
+
+ tu_cs_init_external(sub_cs, cs->cur, cs->reserved_end);
+ tu_cs_begin(sub_cs);
+ result = tu_cs_reserve_space(sub_cs, size);
+ assert(result == VK_SUCCESS);
+
+ return VK_SUCCESS;
}
+/**
+ * Allocate count*size dwords, aligned to size dwords.
+ * \a cs must be in TU_CS_MODE_SUB_STREAM mode.
+ *
+ */
VkResult
-tu_cs_check_space(struct tu_device *dev, struct tu_cs *cs, size_t size)
+tu_cs_alloc(struct tu_cs *cs,
+ uint32_t count,
+ uint32_t size,
+ struct tu_cs_memory *memory)
{
- if (cs->end - cs->cur >= size)
+ assert(cs->mode == TU_CS_MODE_SUB_STREAM);
+ assert(size && size <= 1024);
+
+ if (!count)
return VK_SUCCESS;
- VkResult result = tu_cs_end(cs);
+ /* TODO: smarter way to deal with alignment? */
+
+ VkResult result = tu_cs_reserve_space(cs, count * size + (size-1));
if (result != VK_SUCCESS)
return result;
- return tu_cs_begin(dev, cs, size);
+ struct tu_bo *bo = cs->bos[cs->bo_count - 1];
+ size_t offset = align(tu_cs_get_offset(cs), size);
+
+ memory->map = bo->map + offset * sizeof(uint32_t);
+ memory->iova = bo->iova + offset * sizeof(uint32_t);
+
+ cs->start = cs->cur = (uint32_t*) bo->map + offset + count * size;
+
+ return VK_SUCCESS;
+}
+
+/**
+ * End command packet emission to a sub-stream. \a sub_cs becomes invalid
+ * after this call.
+ *
+ * Return an IB entry for the sub-stream. The entry has the same lifetime as
+ * \a cs.
+ */
+struct tu_cs_entry
+tu_cs_end_sub_stream(struct tu_cs *cs, struct tu_cs *sub_cs)
+{
+ assert(cs->mode == TU_CS_MODE_SUB_STREAM);
+ assert(cs->bo_count);
+ assert(sub_cs->start == cs->cur && sub_cs->end == cs->reserved_end);
+ tu_cs_sanity_check(sub_cs);
+
+ tu_cs_end(sub_cs);
+
+ cs->cur = sub_cs->cur;
+
+ struct tu_cs_entry entry = {
+ .bo = cs->bos[cs->bo_count - 1],
+ .size = tu_cs_get_size(cs) * sizeof(uint32_t),
+ .offset = tu_cs_get_offset(cs) * sizeof(uint32_t),
+ };
+
+ cs->start = cs->cur;
+
+ return entry;
+}
+
+/**
+ * Reserve space from a command stream for \a reserved_size uint32_t values.
+ * This never fails when \a cs has mode TU_CS_MODE_EXTERNAL.
+ */
+VkResult
+tu_cs_reserve_space(struct tu_cs *cs, uint32_t reserved_size)
+{
+ if (tu_cs_get_space(cs) < reserved_size) {
+ if (cs->mode == TU_CS_MODE_EXTERNAL) {
+ unreachable("cannot grow external buffer");
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+ }
+
+ /* add an entry for the exiting command packets */
+ if (!tu_cs_is_empty(cs)) {
+ /* no direct command packet for TU_CS_MODE_SUB_STREAM */
+ assert(cs->mode != TU_CS_MODE_SUB_STREAM);
+
+ tu_cs_add_entry(cs);
+ }
+
+ if (cs->cond_flags) {
+ /* Subtract one here to account for the DWORD field itself. */
+ *cs->cond_dwords = cs->cur - cs->cond_dwords - 1;
+
+ /* space for CP_COND_REG_EXEC in next bo */
+ reserved_size += 3;
+ }
+
+ /* switch to a new BO */
+ uint32_t new_size = MAX2(cs->next_bo_size, reserved_size);
+ VkResult result = tu_cs_add_bo(cs, new_size);
+ if (result != VK_SUCCESS)
+ return result;
+
+ /* if inside a condition, emit a new CP_COND_REG_EXEC */
+ if (cs->cond_flags) {
+ cs->reserved_end = cs->cur + reserved_size;
+
+ tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
+ tu_cs_emit(cs, cs->cond_flags);
+
+ cs->cond_dwords = cs->cur;
+
+ /* Emit dummy DWORD field here */
+ tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(0));
+ }
+
+ /* double the size for the next bo */
+ new_size <<= 1;
+ if (cs->next_bo_size < new_size)
+ cs->next_bo_size = new_size;
+ }
+
+ assert(tu_cs_get_space(cs) >= reserved_size);
+ cs->reserved_end = cs->cur + reserved_size;
+
+ if (cs->mode == TU_CS_MODE_GROW) {
+ /* reserve an entry for the next call to this function or tu_cs_end */
+ return tu_cs_reserve_entry(cs);
+ }
+
+ return VK_SUCCESS;
+}
+
+/**
+ * Reset a command stream to its initial state. This discards all comand
+ * packets in \a cs, but does not necessarily release all resources.
+ */
+void
+tu_cs_reset(struct tu_cs *cs)
+{
+ if (cs->mode == TU_CS_MODE_EXTERNAL) {
+ assert(!cs->bo_count && !cs->entry_count);
+ cs->reserved_end = cs->cur = cs->start;
+ return;
+ }
+
+ for (uint32_t i = 0; i + 1 < cs->bo_count; ++i) {
+ tu_bo_finish(cs->device, cs->bos[i]);
+ free(cs->bos[i]);
+ }
+
+ if (cs->bo_count) {
+ cs->bos[0] = cs->bos[cs->bo_count - 1];
+ cs->bo_count = 1;
+
+ cs->start = cs->cur = cs->reserved_end = (uint32_t *) cs->bos[0]->map;
+ cs->end = cs->start + cs->bos[0]->size / sizeof(uint32_t);
+ }
+
+ cs->entry_count = 0;
}
projects / mesa.git / blobdiff