src/freedreno/vulkan/tu_cs.c

   1 /*
   2  * Copyright © 2019 Google LLC
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include "tu_cs.h"
  25
  26 /**
  27  * Initialize a command stream.
  28  */
  29 void
  30 tu_cs_init(struct tu_cs *cs, enum tu_cs_mode mode, uint32_t initial_size)
  31 {
  32    assert(mode != TU_CS_MODE_EXTERNAL);
  33
  34    memset(cs, 0, sizeof(*cs));
  35
  36    cs->mode = mode;
  37    cs->next_bo_size = initial_size;
  38 }
  39
  40 /**
  41  * Initialize a command stream as a wrapper to an external buffer.
  42  */
  43 void
  44 tu_cs_init_external(struct tu_cs *cs, uint32_t *start, uint32_t *end)
  45 {
  46    memset(cs, 0, sizeof(*cs));
  47
  48    cs->mode = TU_CS_MODE_EXTERNAL;
  49    cs->start = cs->reserved_end = cs->cur = start;
  50    cs->end = end;
  51 }
  52
  53 /**
  54  * Finish and release all resources owned by a command stream.
  55  */
  56 void
  57 tu_cs_finish(struct tu_device *dev, struct tu_cs *cs)
  58 {
  59    for (uint32_t i = 0; i < cs->bo_count; ++i) {
  60       tu_bo_finish(dev, cs->bos[i]);
  61       free(cs->bos[i]);
  62    }
  63
  64    free(cs->entries);
  65    free(cs->bos);
  66 }
  67
  68 /**
  69  * Get the offset of the command packets emitted since the last call to
  70  * tu_cs_add_entry.
  71  */
  72 static uint32_t
  73 tu_cs_get_offset(const struct tu_cs *cs)
  74 {
  75    assert(cs->bo_count);
  76    return cs->start - (uint32_t *) cs->bos[cs->bo_count - 1]->map;
  77 }
  78
  79 /**
  80  * Get the size of the command packets emitted since the last call to
  81  * tu_cs_add_entry.
  82  */
  83 static uint32_t
  84 tu_cs_get_size(const struct tu_cs *cs)
  85 {
  86    return cs->cur - cs->start;
  87 }
  88
  89 /**
  90  * Get the size of the remaining space in the current BO.
  91  */
  92 static uint32_t
  93 tu_cs_get_space(const struct tu_cs *cs)
  94 {
  95    return cs->end - cs->cur;
  96 }
  97
  98 /**
  99  * Return true if there is no command packet emitted since the last call to
 100  * tu_cs_add_entry.
 101  */
 102 static uint32_t
 103 tu_cs_is_empty(const struct tu_cs *cs)
 104 {
 105    return tu_cs_get_size(cs) == 0;
 106 }
 107
 108 /*
 109  * Allocate and add a BO to a command stream.  Following command packets will
 110  * be emitted to the new BO.
 111  */
 112 static VkResult
 113 tu_cs_add_bo(struct tu_device *dev, struct tu_cs *cs, uint32_t size)
 114 {
 115    /* no BO for TU_CS_MODE_EXTERNAL */
 116    assert(cs->mode != TU_CS_MODE_EXTERNAL);
 117
 118    /* no dangling command packet */
 119    assert(tu_cs_is_empty(cs));
 120
 121    /* grow cs->bos if needed */
 122    if (cs->bo_count == cs->bo_capacity) {
 123       uint32_t new_capacity = MAX2(4, 2 * cs->bo_capacity);
 124       struct tu_bo **new_bos =
 125          realloc(cs->bos, new_capacity * sizeof(struct tu_bo *));
 126       if (!new_bos)
 127          return VK_ERROR_OUT_OF_HOST_MEMORY;
 128
 129       cs->bo_capacity = new_capacity;
 130       cs->bos = new_bos;
 131    }
 132
 133    struct tu_bo *new_bo = malloc(sizeof(struct tu_bo));
 134    if (!new_bo)
 135       return VK_ERROR_OUT_OF_HOST_MEMORY;
 136
 137    VkResult result = tu_bo_init_new(dev, new_bo, size * sizeof(uint32_t));
 138    if (result != VK_SUCCESS) {
 139       free(new_bo);
 140       return result;
 141    }
 142
 143    result = tu_bo_map(dev, new_bo);
 144    if (result != VK_SUCCESS) {
 145       tu_bo_finish(dev, new_bo);
 146       free(new_bo);
 147       return result;
 148    }
 149
 150    cs->bos[cs->bo_count++] = new_bo;
 151
 152    cs->start = cs->cur = cs->reserved_end = (uint32_t *) new_bo->map;
 153    cs->end = cs->start + new_bo->size / sizeof(uint32_t);
 154
 155    return VK_SUCCESS;
 156 }
 157
 158 /**
 159  * Reserve an IB entry.
 160  */
 161 static VkResult
 162 tu_cs_reserve_entry(struct tu_cs *cs)
 163 {
 164    /* entries are only for TU_CS_MODE_GROW */
 165    assert(cs->mode == TU_CS_MODE_GROW);
 166
 167    /* grow cs->entries if needed */
 168    if (cs->entry_count == cs->entry_capacity) {
 169       uint32_t new_capacity = MAX2(4, cs->entry_capacity * 2);
 170       struct tu_cs_entry *new_entries =
 171          realloc(cs->entries, new_capacity * sizeof(struct tu_cs_entry));
 172       if (!new_entries)
 173          return VK_ERROR_OUT_OF_HOST_MEMORY;
 174
 175       cs->entry_capacity = new_capacity;
 176       cs->entries = new_entries;
 177    }
 178
 179    return VK_SUCCESS;
 180 }
 181
 182 /**
 183  * Add an IB entry for the command packets emitted since the last call to this
 184  * function.
 185  */
 186 static void
 187 tu_cs_add_entry(struct tu_cs *cs)
 188 {
 189    /* entries are only for TU_CS_MODE_GROW */
 190    assert(cs->mode == TU_CS_MODE_GROW);
 191
 192    /* disallow empty entry */
 193    assert(!tu_cs_is_empty(cs));
 194
 195    /*
 196     * because we disallow empty entry, tu_cs_add_bo and tu_cs_reserve_entry
 197     * must both have been called
 198     */
 199    assert(cs->bo_count);
 200    assert(cs->entry_count < cs->entry_capacity);
 201
 202    /* add an entry for [cs->start, cs->cur] */
 203    cs->entries[cs->entry_count++] = (struct tu_cs_entry) {
 204       .bo = cs->bos[cs->bo_count - 1],
 205       .size = tu_cs_get_size(cs) * sizeof(uint32_t),
 206       .offset = tu_cs_get_offset(cs) * sizeof(uint32_t),
 207    };
 208
 209    cs->start = cs->cur;
 210 }
 211
 212 /**
 213  * same behavior as tu_cs_emit_call but without the indirect
 214  */
 215 VkResult
 216 tu_cs_add_entries(struct tu_cs *cs, struct tu_cs *target)
 217 {
 218    VkResult result;
 219
 220    assert(cs->mode == TU_CS_MODE_GROW);
 221    assert(target->mode == TU_CS_MODE_GROW);
 222
 223    if (!tu_cs_is_empty(cs))
 224       tu_cs_add_entry(cs);
 225
 226    for (unsigned i = 0; i < target->entry_count; i++) {
 227       result = tu_cs_reserve_entry(cs);
 228       if (result != VK_SUCCESS)
 229          return result;
 230       cs->entries[cs->entry_count++] = target->entries[i];
 231    }
 232
 233    return VK_SUCCESS;
 234 }
 235
 236 /**
 237  * Begin (or continue) command packet emission.  This does nothing but sanity
 238  * checks currently.  \a cs must not be in TU_CS_MODE_SUB_STREAM mode.
 239  */
 240 void
 241 tu_cs_begin(struct tu_cs *cs)
 242 {
 243    assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 244    assert(tu_cs_is_empty(cs));
 245 }
 246
 247 /**
 248  * End command packet emission.  This adds an IB entry when \a cs is in
 249  * TU_CS_MODE_GROW mode.
 250  */
 251 void
 252 tu_cs_end(struct tu_cs *cs)
 253 {
 254    assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 255
 256    if (cs->mode == TU_CS_MODE_GROW && !tu_cs_is_empty(cs))
 257       tu_cs_add_entry(cs);
 258 }
 259
 260 /**
 261  * Begin command packet emission to a sub-stream.  \a cs must be in
 262  * TU_CS_MODE_SUB_STREAM mode.
 263  *
 264  * Return \a sub_cs which is in TU_CS_MODE_EXTERNAL mode.  tu_cs_begin and
 265  * tu_cs_reserve_space are implied and \a sub_cs is ready for command packet
 266  * emission.
 267  */
 268 VkResult
 269 tu_cs_begin_sub_stream(struct tu_device *dev,
 270                        struct tu_cs *cs,
 271                        uint32_t size,
 272                        struct tu_cs *sub_cs)
 273 {
 274    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 275    assert(size);
 276
 277    VkResult result = tu_cs_reserve_space(dev, cs, size);
 278    if (result != VK_SUCCESS)
 279       return result;
 280
 281    tu_cs_init_external(sub_cs, cs->cur, cs->reserved_end);
 282    tu_cs_begin(sub_cs);
 283    result = tu_cs_reserve_space(dev, sub_cs, size);
 284    assert(result == VK_SUCCESS);
 285
 286    return VK_SUCCESS;
 287 }
 288
 289 /**
 290  * Allocate count*size dwords, aligned to size dwords.
 291  * \a cs must be in TU_CS_MODE_SUB_STREAM mode.
 292  *
 293  */
 294 VkResult
 295 tu_cs_alloc(struct tu_device *dev,
 296             struct tu_cs *cs,
 297             uint32_t count,
 298             uint32_t size,
 299             struct ts_cs_memory *memory)
 300 {
 301    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 302    assert(size && size <= 1024);
 303
 304    if (!count)
 305       return VK_SUCCESS;
 306
 307    /* TODO: smarter way to deal with alignment? */
 308
 309    VkResult result = tu_cs_reserve_space(dev, cs, count * size + (size-1));
 310    if (result != VK_SUCCESS)
 311       return result;
 312
 313    struct tu_bo *bo = cs->bos[cs->bo_count - 1];
 314    size_t offset = align(tu_cs_get_offset(cs), size);
 315
 316    memory->map = bo->map + offset * sizeof(uint32_t);
 317    memory->iova = bo->iova + offset * sizeof(uint32_t);
 318
 319    cs->start = cs->cur = (uint32_t*) bo->map + offset + count * size;
 320
 321    return VK_SUCCESS;
 322 }
 323
 324 /**
 325  * End command packet emission to a sub-stream.  \a sub_cs becomes invalid
 326  * after this call.
 327  *
 328  * Return an IB entry for the sub-stream.  The entry has the same lifetime as
 329  * \a cs.
 330  */
 331 struct tu_cs_entry
 332 tu_cs_end_sub_stream(struct tu_cs *cs, struct tu_cs *sub_cs)
 333 {
 334    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 335    assert(cs->bo_count);
 336    assert(sub_cs->start == cs->cur && sub_cs->end == cs->reserved_end);
 337    tu_cs_sanity_check(sub_cs);
 338
 339    tu_cs_end(sub_cs);
 340
 341    cs->cur = sub_cs->cur;
 342
 343    struct tu_cs_entry entry = {
 344       .bo = cs->bos[cs->bo_count - 1],
 345       .size = tu_cs_get_size(cs) * sizeof(uint32_t),
 346       .offset = tu_cs_get_offset(cs) * sizeof(uint32_t),
 347    };
 348
 349    cs->start = cs->cur;
 350
 351    return entry;
 352 }
 353
 354 /**
 355  * Reserve space from a command stream for \a reserved_size uint32_t values.
 356  * This never fails when \a cs has mode TU_CS_MODE_EXTERNAL.
 357  */
 358 VkResult
 359 tu_cs_reserve_space(struct tu_device *dev,
 360                     struct tu_cs *cs,
 361                     uint32_t reserved_size)
 362 {
 363    if (tu_cs_get_space(cs) < reserved_size) {
 364       if (cs->mode == TU_CS_MODE_EXTERNAL) {
 365          unreachable("cannot grow external buffer");
 366          return VK_ERROR_OUT_OF_HOST_MEMORY;
 367       }
 368
 369       /* add an entry for the exiting command packets */
 370       if (!tu_cs_is_empty(cs)) {
 371          /* no direct command packet for TU_CS_MODE_SUB_STREAM */
 372          assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 373
 374          tu_cs_add_entry(cs);
 375       }
 376
 377       /* switch to a new BO */
 378       uint32_t new_size = MAX2(cs->next_bo_size, reserved_size);
 379       VkResult result = tu_cs_add_bo(dev, cs, new_size);
 380       if (result != VK_SUCCESS)
 381          return result;
 382
 383       /* double the size for the next bo */
 384       new_size <<= 1;
 385       if (cs->next_bo_size < new_size)
 386          cs->next_bo_size = new_size;
 387    }
 388
 389    assert(tu_cs_get_space(cs) >= reserved_size);
 390    cs->reserved_end = cs->cur + reserved_size;
 391
 392    if (cs->mode == TU_CS_MODE_GROW) {
 393       /* reserve an entry for the next call to this function or tu_cs_end */
 394       return tu_cs_reserve_entry(cs);
 395    }
 396
 397    return VK_SUCCESS;
 398 }
 399
 400 /**
 401  * Reset a command stream to its initial state.  This discards all comand
 402  * packets in \a cs, but does not necessarily release all resources.
 403  */
 404 void
 405 tu_cs_reset(struct tu_device *dev, struct tu_cs *cs)
 406 {
 407    if (cs->mode == TU_CS_MODE_EXTERNAL) {
 408       assert(!cs->bo_count && !cs->entry_count);
 409       cs->reserved_end = cs->cur = cs->start;
 410       return;
 411    }
 412
 413    for (uint32_t i = 0; i + 1 < cs->bo_count; ++i) {
 414       tu_bo_finish(dev, cs->bos[i]);
 415       free(cs->bos[i]);
 416    }
 417
 418    if (cs->bo_count) {
 419       cs->bos[0] = cs->bos[cs->bo_count - 1];
 420       cs->bo_count = 1;
 421
 422       cs->start = cs->cur = cs->reserved_end = (uint32_t *) cs->bos[0]->map;
 423       cs->end = cs->start + cs->bos[0]->size / sizeof(uint32_t);
 424    }
 425
 426    cs->entry_count = 0;
 427 }