src/freedreno/vulkan/tu_cs.c

   1 /*
   2  * Copyright © 2019 Google LLC
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include "tu_cs.h"
  25
  26 /**
  27  * Initialize a command stream.
  28  */
  29 void
  30 tu_cs_init(struct tu_cs *cs, enum tu_cs_mode mode, uint32_t initial_size)
  31 {
  32    assert(mode != TU_CS_MODE_EXTERNAL);
  33
  34    memset(cs, 0, sizeof(*cs));
  35
  36    cs->mode = mode;
  37    cs->next_bo_size = initial_size;
  38 }
  39
  40 /**
  41  * Initialize a command stream as a wrapper to an external buffer.
  42  */
  43 void
  44 tu_cs_init_external(struct tu_cs *cs, uint32_t *start, uint32_t *end)
  45 {
  46    memset(cs, 0, sizeof(*cs));
  47
  48    cs->mode = TU_CS_MODE_EXTERNAL;
  49    cs->start = cs->reserved_end = cs->cur = start;
  50    cs->end = end;
  51 }
  52
  53 /**
  54  * Finish and release all resources owned by a command stream.
  55  */
  56 void
  57 tu_cs_finish(struct tu_device *dev, struct tu_cs *cs)
  58 {
  59    for (uint32_t i = 0; i < cs->bo_count; ++i) {
  60       tu_bo_finish(dev, cs->bos[i]);
  61       free(cs->bos[i]);
  62    }
  63
  64    free(cs->entries);
  65    free(cs->bos);
  66 }
  67
  68 /**
  69  * Get the offset of the command packets emitted since the last call to
  70  * tu_cs_add_entry.
  71  */
  72 static uint32_t
  73 tu_cs_get_offset(const struct tu_cs *cs)
  74 {
  75    assert(cs->bo_count);
  76    return cs->start - (uint32_t *) cs->bos[cs->bo_count - 1]->map;
  77 }
  78
  79 /**
  80  * Get the size of the command packets emitted since the last call to
  81  * tu_cs_add_entry.
  82  */
  83 static uint32_t
  84 tu_cs_get_size(const struct tu_cs *cs)
  85 {
  86    return cs->cur - cs->start;
  87 }
  88
  89 /**
  90  * Get the size of the remaining space in the current BO.
  91  */
  92 static uint32_t
  93 tu_cs_get_space(const struct tu_cs *cs)
  94 {
  95    return cs->end - cs->cur;
  96 }
  97
  98 /**
  99  * Return true if there is no command packet emitted since the last call to
 100  * tu_cs_add_entry.
 101  */
 102 static uint32_t
 103 tu_cs_is_empty(const struct tu_cs *cs)
 104 {
 105    return tu_cs_get_size(cs) == 0;
 106 }
 107
 108 /*
 109  * Allocate and add a BO to a command stream.  Following command packets will
 110  * be emitted to the new BO.
 111  */
 112 static VkResult
 113 tu_cs_add_bo(struct tu_device *dev, struct tu_cs *cs, uint32_t size)
 114 {
 115    /* no BO for TU_CS_MODE_EXTERNAL */
 116    assert(cs->mode != TU_CS_MODE_EXTERNAL);
 117
 118    /* no dangling command packet */
 119    assert(tu_cs_is_empty(cs));
 120
 121    /* grow cs->bos if needed */
 122    if (cs->bo_count == cs->bo_capacity) {
 123       uint32_t new_capacity = MAX2(4, 2 * cs->bo_capacity);
 124       struct tu_bo **new_bos =
 125          realloc(cs->bos, new_capacity * sizeof(struct tu_bo *));
 126       if (!new_bos)
 127          return VK_ERROR_OUT_OF_HOST_MEMORY;
 128
 129       cs->bo_capacity = new_capacity;
 130       cs->bos = new_bos;
 131    }
 132
 133    struct tu_bo *new_bo = malloc(sizeof(struct tu_bo));
 134    if (!new_bo)
 135       return VK_ERROR_OUT_OF_HOST_MEMORY;
 136
 137    VkResult result = tu_bo_init_new(dev, new_bo, size * sizeof(uint32_t));
 138    if (result != VK_SUCCESS) {
 139       free(new_bo);
 140       return result;
 141    }
 142
 143    result = tu_bo_map(dev, new_bo);
 144    if (result != VK_SUCCESS) {
 145       tu_bo_finish(dev, new_bo);
 146       free(new_bo);
 147       return result;
 148    }
 149
 150    cs->bos[cs->bo_count++] = new_bo;
 151
 152    cs->start = cs->cur = cs->reserved_end = (uint32_t *) new_bo->map;
 153    cs->end = cs->start + new_bo->size / sizeof(uint32_t);
 154
 155    return VK_SUCCESS;
 156 }
 157
 158 /**
 159  * Reserve an IB entry.
 160  */
 161 static VkResult
 162 tu_cs_reserve_entry(struct tu_cs *cs)
 163 {
 164    /* entries are only for TU_CS_MODE_GROW */
 165    assert(cs->mode == TU_CS_MODE_GROW);
 166
 167    /* grow cs->entries if needed */
 168    if (cs->entry_count == cs->entry_capacity) {
 169       uint32_t new_capacity = MAX2(4, cs->entry_capacity * 2);
 170       struct tu_cs_entry *new_entries =
 171          realloc(cs->entries, new_capacity * sizeof(struct tu_cs_entry));
 172       if (!new_entries)
 173          return VK_ERROR_OUT_OF_HOST_MEMORY;
 174
 175       cs->entry_capacity = new_capacity;
 176       cs->entries = new_entries;
 177    }
 178
 179    return VK_SUCCESS;
 180 }
 181
 182 /**
 183  * Add an IB entry for the command packets emitted since the last call to this
 184  * function.
 185  */
 186 static void
 187 tu_cs_add_entry(struct tu_cs *cs)
 188 {
 189    /* entries are only for TU_CS_MODE_GROW */
 190    assert(cs->mode == TU_CS_MODE_GROW);
 191
 192    /* disallow empty entry */
 193    assert(!tu_cs_is_empty(cs));
 194
 195    /*
 196     * because we disallow empty entry, tu_cs_add_bo and tu_cs_reserve_entry
 197     * must both have been called
 198     */
 199    assert(cs->bo_count);
 200    assert(cs->entry_count < cs->entry_capacity);
 201
 202    /* add an entry for [cs->start, cs->cur] */
 203    cs->entries[cs->entry_count++] = (struct tu_cs_entry) {
 204       .bo = cs->bos[cs->bo_count - 1],
 205       .size = tu_cs_get_size(cs) * sizeof(uint32_t),
 206       .offset = tu_cs_get_offset(cs) * sizeof(uint32_t),
 207    };
 208
 209    cs->start = cs->cur;
 210 }
 211
 212 /**
 213  * same behavior as tu_cs_emit_call but without the indirect
 214  */
 215 VkResult
 216 tu_cs_add_entries(struct tu_cs *cs, struct tu_cs *target)
 217 {
 218    VkResult result;
 219
 220    assert(cs->mode == TU_CS_MODE_GROW);
 221    assert(target->mode == TU_CS_MODE_GROW);
 222
 223    if (!tu_cs_is_empty(cs)) {
 224       tu_cs_add_entry(cs);
 225       result = tu_cs_reserve_entry(cs);
 226       if (result != VK_SUCCESS)
 227          return result;
 228    }
 229
 230    for (unsigned i = 0; i < target->entry_count; i++) {
 231       cs->entries[cs->entry_count++] = target->entries[i];
 232       result = tu_cs_reserve_entry(cs);
 233       if (result != VK_SUCCESS)
 234          return result;
 235    }
 236
 237    return VK_SUCCESS;
 238 }
 239
 240 /**
 241  * Begin (or continue) command packet emission.  This does nothing but sanity
 242  * checks currently.  \a cs must not be in TU_CS_MODE_SUB_STREAM mode.
 243  */
 244 void
 245 tu_cs_begin(struct tu_cs *cs)
 246 {
 247    assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 248    assert(tu_cs_is_empty(cs));
 249 }
 250
 251 /**
 252  * End command packet emission.  This adds an IB entry when \a cs is in
 253  * TU_CS_MODE_GROW mode.
 254  */
 255 void
 256 tu_cs_end(struct tu_cs *cs)
 257 {
 258    assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 259
 260    if (cs->mode == TU_CS_MODE_GROW && !tu_cs_is_empty(cs))
 261       tu_cs_add_entry(cs);
 262 }
 263
 264 /**
 265  * Begin command packet emission to a sub-stream.  \a cs must be in
 266  * TU_CS_MODE_SUB_STREAM mode.
 267  *
 268  * Return \a sub_cs which is in TU_CS_MODE_EXTERNAL mode.  tu_cs_begin and
 269  * tu_cs_reserve_space are implied and \a sub_cs is ready for command packet
 270  * emission.
 271  */
 272 VkResult
 273 tu_cs_begin_sub_stream(struct tu_device *dev,
 274                        struct tu_cs *cs,
 275                        uint32_t size,
 276                        struct tu_cs *sub_cs)
 277 {
 278    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 279    assert(size);
 280
 281    VkResult result = tu_cs_reserve_space(dev, cs, size);
 282    if (result != VK_SUCCESS)
 283       return result;
 284
 285    tu_cs_init_external(sub_cs, cs->cur, cs->reserved_end);
 286    tu_cs_begin(sub_cs);
 287    result = tu_cs_reserve_space(dev, sub_cs, size);
 288    assert(result == VK_SUCCESS);
 289
 290    return VK_SUCCESS;
 291 }
 292
 293 /**
 294  * Allocate count*size dwords, aligned to size dwords.
 295  * \a cs must be in TU_CS_MODE_SUB_STREAM mode.
 296  *
 297  */
 298 VkResult
 299 tu_cs_alloc(struct tu_device *dev,
 300             struct tu_cs *cs,
 301             uint32_t count,
 302             uint32_t size,
 303             struct ts_cs_memory *memory)
 304 {
 305    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 306    assert(size && size <= 1024);
 307
 308    if (!count)
 309       return VK_SUCCESS;
 310
 311    /* TODO: smarter way to deal with alignment? */
 312
 313    VkResult result = tu_cs_reserve_space(dev, cs, count * size + (size-1));
 314    if (result != VK_SUCCESS)
 315       return result;
 316
 317    struct tu_bo *bo = cs->bos[cs->bo_count - 1];
 318    size_t offset = align(tu_cs_get_offset(cs), size);
 319
 320    memory->map = bo->map + offset * sizeof(uint32_t);
 321    memory->iova = bo->iova + offset * sizeof(uint32_t);
 322
 323    cs->start = cs->cur = (uint32_t*) bo->map + offset + count * size;
 324
 325    return VK_SUCCESS;
 326 }
 327
 328 /**
 329  * End command packet emission to a sub-stream.  \a sub_cs becomes invalid
 330  * after this call.
 331  *
 332  * Return an IB entry for the sub-stream.  The entry has the same lifetime as
 333  * \a cs.
 334  */
 335 struct tu_cs_entry
 336 tu_cs_end_sub_stream(struct tu_cs *cs, struct tu_cs *sub_cs)
 337 {
 338    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 339    assert(cs->bo_count);
 340    assert(sub_cs->start == cs->cur && sub_cs->end == cs->reserved_end);
 341    tu_cs_sanity_check(sub_cs);
 342
 343    tu_cs_end(sub_cs);
 344
 345    cs->cur = sub_cs->cur;
 346
 347    struct tu_cs_entry entry = {
 348       .bo = cs->bos[cs->bo_count - 1],
 349       .size = tu_cs_get_size(cs) * sizeof(uint32_t),
 350       .offset = tu_cs_get_offset(cs) * sizeof(uint32_t),
 351    };
 352
 353    cs->start = cs->cur;
 354
 355    return entry;
 356 }
 357
 358 /**
 359  * Reserve space from a command stream for \a reserved_size uint32_t values.
 360  * This never fails when \a cs has mode TU_CS_MODE_EXTERNAL.
 361  */
 362 VkResult
 363 tu_cs_reserve_space(struct tu_device *dev,
 364                     struct tu_cs *cs,
 365                     uint32_t reserved_size)
 366 {
 367    if (tu_cs_get_space(cs) < reserved_size) {
 368       if (cs->mode == TU_CS_MODE_EXTERNAL) {
 369          unreachable("cannot grow external buffer");
 370          return VK_ERROR_OUT_OF_HOST_MEMORY;
 371       }
 372
 373       /* add an entry for the exiting command packets */
 374       if (!tu_cs_is_empty(cs)) {
 375          /* no direct command packet for TU_CS_MODE_SUB_STREAM */
 376          assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 377
 378          tu_cs_add_entry(cs);
 379       }
 380
 381       /* switch to a new BO */
 382       uint32_t new_size = MAX2(cs->next_bo_size, reserved_size);
 383       VkResult result = tu_cs_add_bo(dev, cs, new_size);
 384       if (result != VK_SUCCESS)
 385          return result;
 386
 387       /* double the size for the next bo */
 388       new_size <<= 1;
 389       if (cs->next_bo_size < new_size)
 390          cs->next_bo_size = new_size;
 391    }
 392
 393    assert(tu_cs_get_space(cs) >= reserved_size);
 394    cs->reserved_end = cs->cur + reserved_size;
 395
 396    if (cs->mode == TU_CS_MODE_GROW) {
 397       /* reserve an entry for the next call to this function or tu_cs_end */
 398       return tu_cs_reserve_entry(cs);
 399    }
 400
 401    return VK_SUCCESS;
 402 }
 403
 404 /**
 405  * Reset a command stream to its initial state.  This discards all comand
 406  * packets in \a cs, but does not necessarily release all resources.
 407  */
 408 void
 409 tu_cs_reset(struct tu_device *dev, struct tu_cs *cs)
 410 {
 411    if (cs->mode == TU_CS_MODE_EXTERNAL) {
 412       assert(!cs->bo_count && !cs->entry_count);
 413       cs->reserved_end = cs->cur = cs->start;
 414       return;
 415    }
 416
 417    for (uint32_t i = 0; i + 1 < cs->bo_count; ++i) {
 418       tu_bo_finish(dev, cs->bos[i]);
 419       free(cs->bos[i]);
 420    }
 421
 422    if (cs->bo_count) {
 423       cs->bos[0] = cs->bos[cs->bo_count - 1];
 424       cs->bo_count = 1;
 425
 426       cs->start = cs->cur = cs->reserved_end = (uint32_t *) cs->bos[0]->map;
 427       cs->end = cs->start + cs->bos[0]->size / sizeof(uint32_t);
 428    }
 429
 430    cs->entry_count = 0;
 431 }