src/freedreno/vulkan/tu_cs.c

   1 /*
   2  * Copyright © 2019 Google LLC
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include "tu_cs.h"
  25
  26 /**
  27  * Initialize a command stream.
  28  */
  29 void
  30 tu_cs_init(struct tu_cs *cs,
  31            struct tu_device *device,
  32            enum tu_cs_mode mode,
  33            uint32_t initial_size)
  34 {
  35    assert(mode != TU_CS_MODE_EXTERNAL);
  36
  37    memset(cs, 0, sizeof(*cs));
  38
  39    cs->device = device;
  40    cs->mode = mode;
  41    cs->next_bo_size = initial_size;
  42 }
  43
  44 /**
  45  * Initialize a command stream as a wrapper to an external buffer.
  46  */
  47 void
  48 tu_cs_init_external(struct tu_cs *cs, uint32_t *start, uint32_t *end)
  49 {
  50    memset(cs, 0, sizeof(*cs));
  51
  52    cs->mode = TU_CS_MODE_EXTERNAL;
  53    cs->start = cs->reserved_end = cs->cur = start;
  54    cs->end = end;
  55 }
  56
  57 /**
  58  * Finish and release all resources owned by a command stream.
  59  */
  60 void
  61 tu_cs_finish(struct tu_cs *cs)
  62 {
  63    for (uint32_t i = 0; i < cs->bo_count; ++i) {
  64       tu_bo_finish(cs->device, cs->bos[i]);
  65       free(cs->bos[i]);
  66    }
  67
  68    free(cs->entries);
  69    free(cs->bos);
  70 }
  71
  72 /**
  73  * Get the offset of the command packets emitted since the last call to
  74  * tu_cs_add_entry.
  75  */
  76 static uint32_t
  77 tu_cs_get_offset(const struct tu_cs *cs)
  78 {
  79    assert(cs->bo_count);
  80    return cs->start - (uint32_t *) cs->bos[cs->bo_count - 1]->map;
  81 }
  82
  83 /**
  84  * Get the size of the command packets emitted since the last call to
  85  * tu_cs_add_entry.
  86  */
  87 static uint32_t
  88 tu_cs_get_size(const struct tu_cs *cs)
  89 {
  90    return cs->cur - cs->start;
  91 }
  92
  93 /**
  94  * Return true if there is no command packet emitted since the last call to
  95  * tu_cs_add_entry.
  96  */
  97 static uint32_t
  98 tu_cs_is_empty(const struct tu_cs *cs)
  99 {
 100    return tu_cs_get_size(cs) == 0;
 101 }
 102
 103 /*
 104  * Allocate and add a BO to a command stream.  Following command packets will
 105  * be emitted to the new BO.
 106  */
 107 static VkResult
 108 tu_cs_add_bo(struct tu_cs *cs, uint32_t size)
 109 {
 110    /* no BO for TU_CS_MODE_EXTERNAL */
 111    assert(cs->mode != TU_CS_MODE_EXTERNAL);
 112
 113    /* no dangling command packet */
 114    assert(tu_cs_is_empty(cs));
 115
 116    /* grow cs->bos if needed */
 117    if (cs->bo_count == cs->bo_capacity) {
 118       uint32_t new_capacity = MAX2(4, 2 * cs->bo_capacity);
 119       struct tu_bo **new_bos =
 120          realloc(cs->bos, new_capacity * sizeof(struct tu_bo *));
 121       if (!new_bos)
 122          return VK_ERROR_OUT_OF_HOST_MEMORY;
 123
 124       cs->bo_capacity = new_capacity;
 125       cs->bos = new_bos;
 126    }
 127
 128    struct tu_bo *new_bo = malloc(sizeof(struct tu_bo));
 129    if (!new_bo)
 130       return VK_ERROR_OUT_OF_HOST_MEMORY;
 131
 132    VkResult result =
 133       tu_bo_init_new(cs->device, new_bo, size * sizeof(uint32_t));
 134    if (result != VK_SUCCESS) {
 135       free(new_bo);
 136       return result;
 137    }
 138
 139    result = tu_bo_map(cs->device, new_bo);
 140    if (result != VK_SUCCESS) {
 141       tu_bo_finish(cs->device, new_bo);
 142       free(new_bo);
 143       return result;
 144    }
 145
 146    cs->bos[cs->bo_count++] = new_bo;
 147
 148    cs->start = cs->cur = cs->reserved_end = (uint32_t *) new_bo->map;
 149    cs->end = cs->start + new_bo->size / sizeof(uint32_t);
 150
 151    return VK_SUCCESS;
 152 }
 153
 154 /**
 155  * Reserve an IB entry.
 156  */
 157 static VkResult
 158 tu_cs_reserve_entry(struct tu_cs *cs)
 159 {
 160    /* entries are only for TU_CS_MODE_GROW */
 161    assert(cs->mode == TU_CS_MODE_GROW);
 162
 163    /* grow cs->entries if needed */
 164    if (cs->entry_count == cs->entry_capacity) {
 165       uint32_t new_capacity = MAX2(4, cs->entry_capacity * 2);
 166       struct tu_cs_entry *new_entries =
 167          realloc(cs->entries, new_capacity * sizeof(struct tu_cs_entry));
 168       if (!new_entries)
 169          return VK_ERROR_OUT_OF_HOST_MEMORY;
 170
 171       cs->entry_capacity = new_capacity;
 172       cs->entries = new_entries;
 173    }
 174
 175    return VK_SUCCESS;
 176 }
 177
 178 /**
 179  * Add an IB entry for the command packets emitted since the last call to this
 180  * function.
 181  */
 182 static void
 183 tu_cs_add_entry(struct tu_cs *cs)
 184 {
 185    /* entries are only for TU_CS_MODE_GROW */
 186    assert(cs->mode == TU_CS_MODE_GROW);
 187
 188    /* disallow empty entry */
 189    assert(!tu_cs_is_empty(cs));
 190
 191    /*
 192     * because we disallow empty entry, tu_cs_add_bo and tu_cs_reserve_entry
 193     * must both have been called
 194     */
 195    assert(cs->bo_count);
 196    assert(cs->entry_count < cs->entry_capacity);
 197
 198    /* add an entry for [cs->start, cs->cur] */
 199    cs->entries[cs->entry_count++] = (struct tu_cs_entry) {
 200       .bo = cs->bos[cs->bo_count - 1],
 201       .size = tu_cs_get_size(cs) * sizeof(uint32_t),
 202       .offset = tu_cs_get_offset(cs) * sizeof(uint32_t),
 203    };
 204
 205    cs->start = cs->cur;
 206 }
 207
 208 /**
 209  * same behavior as tu_cs_emit_call but without the indirect
 210  */
 211 VkResult
 212 tu_cs_add_entries(struct tu_cs *cs, struct tu_cs *target)
 213 {
 214    VkResult result;
 215
 216    assert(cs->mode == TU_CS_MODE_GROW);
 217    assert(target->mode == TU_CS_MODE_GROW);
 218
 219    if (!tu_cs_is_empty(cs))
 220       tu_cs_add_entry(cs);
 221
 222    for (unsigned i = 0; i < target->entry_count; i++) {
 223       result = tu_cs_reserve_entry(cs);
 224       if (result != VK_SUCCESS)
 225          return result;
 226       cs->entries[cs->entry_count++] = target->entries[i];
 227    }
 228
 229    return VK_SUCCESS;
 230 }
 231
 232 /**
 233  * Begin (or continue) command packet emission.  This does nothing but sanity
 234  * checks currently.  \a cs must not be in TU_CS_MODE_SUB_STREAM mode.
 235  */
 236 void
 237 tu_cs_begin(struct tu_cs *cs)
 238 {
 239    assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 240    assert(tu_cs_is_empty(cs));
 241 }
 242
 243 /**
 244  * End command packet emission.  This adds an IB entry when \a cs is in
 245  * TU_CS_MODE_GROW mode.
 246  */
 247 void
 248 tu_cs_end(struct tu_cs *cs)
 249 {
 250    assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 251
 252    if (cs->mode == TU_CS_MODE_GROW && !tu_cs_is_empty(cs))
 253       tu_cs_add_entry(cs);
 254 }
 255
 256 /**
 257  * Begin command packet emission to a sub-stream.  \a cs must be in
 258  * TU_CS_MODE_SUB_STREAM mode.
 259  *
 260  * Return \a sub_cs which is in TU_CS_MODE_EXTERNAL mode.  tu_cs_begin and
 261  * tu_cs_reserve_space are implied and \a sub_cs is ready for command packet
 262  * emission.
 263  */
 264 VkResult
 265 tu_cs_begin_sub_stream(struct tu_cs *cs, uint32_t size, struct tu_cs *sub_cs)
 266 {
 267    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 268    assert(size);
 269
 270    VkResult result = tu_cs_reserve_space(cs, size);
 271    if (result != VK_SUCCESS)
 272       return result;
 273
 274    tu_cs_init_external(sub_cs, cs->cur, cs->reserved_end);
 275    tu_cs_begin(sub_cs);
 276    result = tu_cs_reserve_space(sub_cs, size);
 277    assert(result == VK_SUCCESS);
 278
 279    return VK_SUCCESS;
 280 }
 281
 282 /**
 283  * Allocate count*size dwords, aligned to size dwords.
 284  * \a cs must be in TU_CS_MODE_SUB_STREAM mode.
 285  *
 286  */
 287 VkResult
 288 tu_cs_alloc(struct tu_cs *cs,
 289             uint32_t count,
 290             uint32_t size,
 291             struct ts_cs_memory *memory)
 292 {
 293    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 294    assert(size && size <= 1024);
 295
 296    if (!count)
 297       return VK_SUCCESS;
 298
 299    /* TODO: smarter way to deal with alignment? */
 300
 301    VkResult result = tu_cs_reserve_space(cs, count * size + (size-1));
 302    if (result != VK_SUCCESS)
 303       return result;
 304
 305    struct tu_bo *bo = cs->bos[cs->bo_count - 1];
 306    size_t offset = align(tu_cs_get_offset(cs), size);
 307
 308    memory->map = bo->map + offset * sizeof(uint32_t);
 309    memory->iova = bo->iova + offset * sizeof(uint32_t);
 310
 311    cs->start = cs->cur = (uint32_t*) bo->map + offset + count * size;
 312
 313    return VK_SUCCESS;
 314 }
 315
 316 /**
 317  * End command packet emission to a sub-stream.  \a sub_cs becomes invalid
 318  * after this call.
 319  *
 320  * Return an IB entry for the sub-stream.  The entry has the same lifetime as
 321  * \a cs.
 322  */
 323 struct tu_cs_entry
 324 tu_cs_end_sub_stream(struct tu_cs *cs, struct tu_cs *sub_cs)
 325 {
 326    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 327    assert(cs->bo_count);
 328    assert(sub_cs->start == cs->cur && sub_cs->end == cs->reserved_end);
 329    tu_cs_sanity_check(sub_cs);
 330
 331    tu_cs_end(sub_cs);
 332
 333    cs->cur = sub_cs->cur;
 334
 335    struct tu_cs_entry entry = {
 336       .bo = cs->bos[cs->bo_count - 1],
 337       .size = tu_cs_get_size(cs) * sizeof(uint32_t),
 338       .offset = tu_cs_get_offset(cs) * sizeof(uint32_t),
 339    };
 340
 341    cs->start = cs->cur;
 342
 343    return entry;
 344 }
 345
 346 /**
 347  * Reserve space from a command stream for \a reserved_size uint32_t values.
 348  * This never fails when \a cs has mode TU_CS_MODE_EXTERNAL.
 349  */
 350 VkResult
 351 tu_cs_reserve_space(struct tu_cs *cs, uint32_t reserved_size)
 352 {
 353    if (tu_cs_get_space(cs) < reserved_size) {
 354       if (cs->mode == TU_CS_MODE_EXTERNAL) {
 355          unreachable("cannot grow external buffer");
 356          return VK_ERROR_OUT_OF_HOST_MEMORY;
 357       }
 358
 359       /* add an entry for the exiting command packets */
 360       if (!tu_cs_is_empty(cs)) {
 361          /* no direct command packet for TU_CS_MODE_SUB_STREAM */
 362          assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 363
 364          tu_cs_add_entry(cs);
 365       }
 366
 367       if (cs->cond_flags) {
 368          /* Subtract one here to account for the DWORD field itself. */
 369          *cs->cond_dwords = cs->cur - cs->cond_dwords - 1;
 370
 371          /* space for CP_COND_REG_EXEC in next bo */
 372          reserved_size += 3;
 373       }
 374
 375       /* switch to a new BO */
 376       uint32_t new_size = MAX2(cs->next_bo_size, reserved_size);
 377       VkResult result = tu_cs_add_bo(cs, new_size);
 378       if (result != VK_SUCCESS)
 379          return result;
 380
 381       /* if inside a condition, emit a new CP_COND_REG_EXEC */
 382       if (cs->cond_flags) {
 383          cs->reserved_end = cs->cur + reserved_size;
 384
 385          tu_cs_emit_pkt7(cs, CP_COND_REG_EXEC, 2);
 386          tu_cs_emit(cs, cs->cond_flags);
 387
 388          cs->cond_dwords = cs->cur;
 389
 390          /* Emit dummy DWORD field here */
 391          tu_cs_emit(cs, CP_COND_REG_EXEC_1_DWORDS(0));
 392       }
 393
 394       /* double the size for the next bo */
 395       new_size <<= 1;
 396       if (cs->next_bo_size < new_size)
 397          cs->next_bo_size = new_size;
 398    }
 399
 400    assert(tu_cs_get_space(cs) >= reserved_size);
 401    cs->reserved_end = cs->cur + reserved_size;
 402
 403    if (cs->mode == TU_CS_MODE_GROW) {
 404       /* reserve an entry for the next call to this function or tu_cs_end */
 405       return tu_cs_reserve_entry(cs);
 406    }
 407
 408    return VK_SUCCESS;
 409 }
 410
 411 /**
 412  * Reset a command stream to its initial state.  This discards all comand
 413  * packets in \a cs, but does not necessarily release all resources.
 414  */
 415 void
 416 tu_cs_reset(struct tu_cs *cs)
 417 {
 418    if (cs->mode == TU_CS_MODE_EXTERNAL) {
 419       assert(!cs->bo_count && !cs->entry_count);
 420       cs->reserved_end = cs->cur = cs->start;
 421       return;
 422    }
 423
 424    for (uint32_t i = 0; i + 1 < cs->bo_count; ++i) {
 425       tu_bo_finish(cs->device, cs->bos[i]);
 426       free(cs->bos[i]);
 427    }
 428
 429    if (cs->bo_count) {
 430       cs->bos[0] = cs->bos[cs->bo_count - 1];
 431       cs->bo_count = 1;
 432
 433       cs->start = cs->cur = cs->reserved_end = (uint32_t *) cs->bos[0]->map;
 434       cs->end = cs->start + cs->bos[0]->size / sizeof(uint32_t);
 435    }
 436
 437    cs->entry_count = 0;
 438 }