src/freedreno/vulkan/tu_cs.c

   1 /*
   2  * Copyright © 2019 Google LLC
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include "tu_cs.h"
  25
  26 /**
  27  * Initialize a command stream.
  28  */
  29 void
  30 tu_cs_init(struct tu_cs *cs, enum tu_cs_mode mode, uint32_t initial_size)
  31 {
  32    assert(mode != TU_CS_MODE_EXTERNAL);
  33
  34    memset(cs, 0, sizeof(*cs));
  35
  36    cs->mode = mode;
  37    cs->next_bo_size = initial_size;
  38 }
  39
  40 /**
  41  * Initialize a command stream as a wrapper to an external buffer.
  42  */
  43 void
  44 tu_cs_init_external(struct tu_cs *cs, uint32_t *start, uint32_t *end)
  45 {
  46    memset(cs, 0, sizeof(*cs));
  47
  48    cs->mode = TU_CS_MODE_EXTERNAL;
  49    cs->start = cs->reserved_end = cs->cur = start;
  50    cs->end = end;
  51 }
  52
  53 /**
  54  * Finish and release all resources owned by a command stream.
  55  */
  56 void
  57 tu_cs_finish(struct tu_device *dev, struct tu_cs *cs)
  58 {
  59    for (uint32_t i = 0; i < cs->bo_count; ++i) {
  60       tu_bo_finish(dev, cs->bos[i]);
  61       free(cs->bos[i]);
  62    }
  63
  64    free(cs->entries);
  65    free(cs->bos);
  66 }
  67
  68 /**
  69  * Get the offset of the command packets emitted since the last call to
  70  * tu_cs_add_entry.
  71  */
  72 static uint32_t
  73 tu_cs_get_offset(const struct tu_cs *cs)
  74 {
  75    assert(cs->bo_count);
  76    return cs->start - (uint32_t *) cs->bos[cs->bo_count - 1]->map;
  77 }
  78
  79 /**
  80  * Get the size of the command packets emitted since the last call to
  81  * tu_cs_add_entry.
  82  */
  83 static uint32_t
  84 tu_cs_get_size(const struct tu_cs *cs)
  85 {
  86    return cs->cur - cs->start;
  87 }
  88
  89 /**
  90  * Get the size of the remaining space in the current BO.
  91  */
  92 static uint32_t
  93 tu_cs_get_space(const struct tu_cs *cs)
  94 {
  95    return cs->end - cs->cur;
  96 }
  97
  98 /**
  99  * Return true if there is no command packet emitted since the last call to
 100  * tu_cs_add_entry.
 101  */
 102 static uint32_t
 103 tu_cs_is_empty(const struct tu_cs *cs)
 104 {
 105    return tu_cs_get_size(cs) == 0;
 106 }
 107
 108 /*
 109  * Allocate and add a BO to a command stream.  Following command packets will
 110  * be emitted to the new BO.
 111  */
 112 static VkResult
 113 tu_cs_add_bo(struct tu_device *dev, struct tu_cs *cs, uint32_t size)
 114 {
 115    /* no BO for TU_CS_MODE_EXTERNAL */
 116    assert(cs->mode != TU_CS_MODE_EXTERNAL);
 117
 118    /* no dangling command packet */
 119    assert(tu_cs_is_empty(cs));
 120
 121    /* grow cs->bos if needed */
 122    if (cs->bo_count == cs->bo_capacity) {
 123       uint32_t new_capacity = MAX2(4, 2 * cs->bo_capacity);
 124       struct tu_bo **new_bos =
 125          realloc(cs->bos, new_capacity * sizeof(struct tu_bo *));
 126       if (!new_bos)
 127          return VK_ERROR_OUT_OF_HOST_MEMORY;
 128
 129       cs->bo_capacity = new_capacity;
 130       cs->bos = new_bos;
 131    }
 132
 133    struct tu_bo *new_bo = malloc(sizeof(struct tu_bo));
 134    if (!new_bo)
 135       return VK_ERROR_OUT_OF_HOST_MEMORY;
 136
 137    VkResult result = tu_bo_init_new(dev, new_bo, size * sizeof(uint32_t));
 138    if (result != VK_SUCCESS) {
 139       free(new_bo);
 140       return result;
 141    }
 142
 143    result = tu_bo_map(dev, new_bo);
 144    if (result != VK_SUCCESS) {
 145       tu_bo_finish(dev, new_bo);
 146       free(new_bo);
 147       return result;
 148    }
 149
 150    cs->bos[cs->bo_count++] = new_bo;
 151
 152    cs->start = cs->cur = cs->reserved_end = (uint32_t *) new_bo->map;
 153    cs->end = cs->start + new_bo->size / sizeof(uint32_t);
 154
 155    return VK_SUCCESS;
 156 }
 157
 158 /**
 159  * Reserve an IB entry.
 160  */
 161 static VkResult
 162 tu_cs_reserve_entry(struct tu_device *dev, struct tu_cs *cs)
 163 {
 164    /* entries are only for TU_CS_MODE_GROW */
 165    assert(cs->mode == TU_CS_MODE_GROW);
 166
 167    /* grow cs->entries if needed */
 168    if (cs->entry_count == cs->entry_capacity) {
 169       uint32_t new_capacity = MAX2(4, cs->entry_capacity * 2);
 170       struct tu_cs_entry *new_entries =
 171          realloc(cs->entries, new_capacity * sizeof(struct tu_cs_entry));
 172       if (!new_entries)
 173          return VK_ERROR_OUT_OF_HOST_MEMORY;
 174
 175       cs->entry_capacity = new_capacity;
 176       cs->entries = new_entries;
 177    }
 178
 179    return VK_SUCCESS;
 180 }
 181
 182 /**
 183  * Add an IB entry for the command packets emitted since the last call to this
 184  * function.
 185  */
 186 static void
 187 tu_cs_add_entry(struct tu_cs *cs)
 188 {
 189    /* entries are only for TU_CS_MODE_GROW */
 190    assert(cs->mode == TU_CS_MODE_GROW);
 191
 192    /* disallow empty entry */
 193    assert(!tu_cs_is_empty(cs));
 194
 195    /*
 196     * because we disallow empty entry, tu_cs_add_bo and tu_cs_reserve_entry
 197     * must both have been called
 198     */
 199    assert(cs->bo_count);
 200    assert(cs->entry_count < cs->entry_capacity);
 201
 202    /* add an entry for [cs->start, cs->cur] */
 203    cs->entries[cs->entry_count++] = (struct tu_cs_entry) {
 204       .bo = cs->bos[cs->bo_count - 1],
 205       .size = tu_cs_get_size(cs) * sizeof(uint32_t),
 206       .offset = tu_cs_get_offset(cs) * sizeof(uint32_t),
 207    };
 208
 209    cs->start = cs->cur;
 210 }
 211
 212 /**
 213  * Begin (or continue) command packet emission.  This does nothing but sanity
 214  * checks currently.  \a cs must not be in TU_CS_MODE_SUB_STREAM mode.
 215  */
 216 void
 217 tu_cs_begin(struct tu_cs *cs)
 218 {
 219    assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 220    assert(tu_cs_is_empty(cs));
 221 }
 222
 223 /**
 224  * End command packet emission.  This adds an IB entry when \a cs is in
 225  * TU_CS_MODE_GROW mode.
 226  */
 227 void
 228 tu_cs_end(struct tu_cs *cs)
 229 {
 230    assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 231
 232    if (cs->mode == TU_CS_MODE_GROW && !tu_cs_is_empty(cs))
 233       tu_cs_add_entry(cs);
 234 }
 235
 236 /**
 237  * Begin command packet emission to a sub-stream.  \a cs must be in
 238  * TU_CS_MODE_SUB_STREAM mode.
 239  *
 240  * Return \a sub_cs which is in TU_CS_MODE_EXTERNAL mode.  tu_cs_begin and
 241  * tu_cs_reserve_space are implied and \a sub_cs is ready for command packet
 242  * emission.
 243  */
 244 VkResult
 245 tu_cs_begin_sub_stream(struct tu_device *dev,
 246                        struct tu_cs *cs,
 247                        uint32_t size,
 248                        struct tu_cs *sub_cs)
 249 {
 250    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 251    assert(size);
 252
 253    VkResult result = tu_cs_reserve_space(dev, cs, size);
 254    if (result != VK_SUCCESS)
 255       return result;
 256
 257    tu_cs_init_external(sub_cs, cs->cur, cs->reserved_end);
 258    tu_cs_begin(sub_cs);
 259    result = tu_cs_reserve_space(dev, sub_cs, size);
 260    assert(result == VK_SUCCESS);
 261
 262    return VK_SUCCESS;
 263 }
 264
 265 /**
 266  * Allocate count*size dwords, aligned to size dwords.
 267  * \a cs must be in TU_CS_MODE_SUB_STREAM mode.
 268  *
 269  */
 270 VkResult
 271 tu_cs_alloc(struct tu_device *dev,
 272             struct tu_cs *cs,
 273             uint32_t count,
 274             uint32_t size,
 275             struct ts_cs_memory *memory)
 276 {
 277    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 278    assert(size && size <= 1024);
 279
 280    if (!count)
 281       return VK_SUCCESS;
 282
 283    /* TODO: smarter way to deal with alignment? */
 284
 285    VkResult result = tu_cs_reserve_space(dev, cs, count * size + (size-1));
 286    if (result != VK_SUCCESS)
 287       return result;
 288
 289    struct tu_bo *bo = cs->bos[cs->bo_count - 1];
 290    size_t offset = align(tu_cs_get_offset(cs), size);
 291
 292    memory->map = bo->map + offset * sizeof(uint32_t);
 293    memory->iova = bo->iova + offset * sizeof(uint32_t);
 294
 295    cs->start = cs->cur = (uint32_t*) bo->map + offset + count * size;
 296
 297    return VK_SUCCESS;
 298 }
 299
 300 /**
 301  * End command packet emission to a sub-stream.  \a sub_cs becomes invalid
 302  * after this call.
 303  *
 304  * Return an IB entry for the sub-stream.  The entry has the same lifetime as
 305  * \a cs.
 306  */
 307 struct tu_cs_entry
 308 tu_cs_end_sub_stream(struct tu_cs *cs, struct tu_cs *sub_cs)
 309 {
 310    assert(cs->mode == TU_CS_MODE_SUB_STREAM);
 311    assert(cs->bo_count);
 312    assert(sub_cs->start == cs->cur && sub_cs->end == cs->reserved_end);
 313    tu_cs_sanity_check(sub_cs);
 314
 315    tu_cs_end(sub_cs);
 316
 317    cs->cur = sub_cs->cur;
 318
 319    struct tu_cs_entry entry = {
 320       .bo = cs->bos[cs->bo_count - 1],
 321       .size = tu_cs_get_size(cs) * sizeof(uint32_t),
 322       .offset = tu_cs_get_offset(cs) * sizeof(uint32_t),
 323    };
 324
 325    cs->start = cs->cur;
 326
 327    return entry;
 328 }
 329
 330 /**
 331  * Reserve space from a command stream for \a reserved_size uint32_t values.
 332  * This never fails when \a cs has mode TU_CS_MODE_EXTERNAL.
 333  */
 334 VkResult
 335 tu_cs_reserve_space(struct tu_device *dev,
 336                     struct tu_cs *cs,
 337                     uint32_t reserved_size)
 338 {
 339    if (tu_cs_get_space(cs) < reserved_size) {
 340       if (cs->mode == TU_CS_MODE_EXTERNAL) {
 341          unreachable("cannot grow external buffer");
 342          return VK_ERROR_OUT_OF_HOST_MEMORY;
 343       }
 344
 345       /* add an entry for the exiting command packets */
 346       if (!tu_cs_is_empty(cs)) {
 347          /* no direct command packet for TU_CS_MODE_SUB_STREAM */
 348          assert(cs->mode != TU_CS_MODE_SUB_STREAM);
 349
 350          tu_cs_add_entry(cs);
 351       }
 352
 353       /* switch to a new BO */
 354       uint32_t new_size = MAX2(cs->next_bo_size, reserved_size);
 355       VkResult result = tu_cs_add_bo(dev, cs, new_size);
 356       if (result != VK_SUCCESS)
 357          return result;
 358
 359       /* double the size for the next bo */
 360       new_size <<= 1;
 361       if (cs->next_bo_size < new_size)
 362          cs->next_bo_size = new_size;
 363    }
 364
 365    assert(tu_cs_get_space(cs) >= reserved_size);
 366    cs->reserved_end = cs->cur + reserved_size;
 367
 368    if (cs->mode == TU_CS_MODE_GROW) {
 369       /* reserve an entry for the next call to this function or tu_cs_end */
 370       return tu_cs_reserve_entry(dev, cs);
 371    }
 372
 373    return VK_SUCCESS;
 374 }
 375
 376 /**
 377  * Reset a command stream to its initial state.  This discards all comand
 378  * packets in \a cs, but does not necessarily release all resources.
 379  */
 380 void
 381 tu_cs_reset(struct tu_device *dev, struct tu_cs *cs)
 382 {
 383    if (cs->mode == TU_CS_MODE_EXTERNAL) {
 384       assert(!cs->bo_count && !cs->entry_count);
 385       cs->reserved_end = cs->cur = cs->start;
 386       return;
 387    }
 388
 389    for (uint32_t i = 0; i + 1 < cs->bo_count; ++i) {
 390       tu_bo_finish(dev, cs->bos[i]);
 391       free(cs->bos[i]);
 392    }
 393
 394    if (cs->bo_count) {
 395       cs->bos[0] = cs->bos[cs->bo_count - 1];
 396       cs->bo_count = 1;
 397
 398       cs->start = cs->cur = cs->reserved_end = (uint32_t *) cs->bos[0]->map;
 399       cs->end = cs->start + cs->bos[0]->size / sizeof(uint32_t);
 400    }
 401
 402    cs->entry_count = 0;
 403 }