src/freedreno/drm/msm_ringbuffer.c

   1 /*
   2  * Copyright (C) 2012-2018 Rob Clark <robclark@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark <robclark@freedesktop.org>
  25  */
  26
  27 #include <assert.h>
  28 #include <inttypes.h>
  29
  30 #include "util/hash_table.h"
  31 #include "util/set.h"
  32 #include "util/slab.h"
  33
  34 #include "drm/freedreno_ringbuffer.h"
  35 #include "msm_priv.h"
  36
  37 /* The legacy implementation of submit/ringbuffer, which still does the
  38  * traditional reloc and cmd tracking
  39  */
  40
  41
  42 #define INIT_SIZE 0x1000
  43
  44
  45 struct msm_submit {
  46         struct fd_submit base;
  47
  48         DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos);
  49         DECLARE_ARRAY(struct fd_bo *, bos);
  50
  51         /* maps fd_bo to idx in bos table: */
  52         struct hash_table *bo_table;
  53
  54         struct slab_mempool ring_pool;
  55
  56         /* hash-set of associated rings: */
  57         struct set *ring_set;
  58
  59         struct fd_ringbuffer *primary;
  60
  61         /* Allow for sub-allocation of stateobj ring buffers (ie. sharing
  62          * the same underlying bo)..
  63          *
  64          * We also rely on previous stateobj having been fully constructed
  65          * so we can reclaim extra space at it's end.
  66          */
  67         struct fd_ringbuffer *suballoc_ring;
  68 };
  69 FD_DEFINE_CAST(fd_submit, msm_submit);
  70
  71 /* for FD_RINGBUFFER_GROWABLE rb's, tracks the 'finalized' cmdstream buffers
  72  * and sizes.  Ie. a finalized buffer can have no more commands appended to
  73  * it.
  74  */
  75 struct msm_cmd {
  76         struct fd_bo *ring_bo;
  77         unsigned size;
  78         DECLARE_ARRAY(struct drm_msm_gem_submit_reloc, relocs);
  79 };
  80
  81 static struct msm_cmd *
  82 cmd_new(struct fd_bo *ring_bo)
  83 {
  84         struct msm_cmd *cmd = malloc(sizeof(*cmd));
  85         cmd->ring_bo = fd_bo_ref(ring_bo);
  86         cmd->size = 0;
  87         cmd->nr_relocs = cmd->max_relocs = 0;
  88         cmd->relocs = NULL;
  89         return cmd;
  90 }
  91
  92 static void
  93 cmd_free(struct msm_cmd *cmd)
  94 {
  95         fd_bo_del(cmd->ring_bo);
  96         free(cmd->relocs);
  97         free(cmd);
  98 }
  99
 100 /* for _FD_RINGBUFFER_OBJECT rb's we need to track the bo's and flags to
 101  * later copy into the submit when the stateobj rb is later referenced by
 102  * a regular rb:
 103  */
 104 struct msm_reloc_bo {
 105         struct fd_bo *bo;
 106         unsigned flags;
 107 };
 108
 109 struct msm_ringbuffer {
 110         struct fd_ringbuffer base;
 111
 112         /* for FD_RINGBUFFER_STREAMING rb's which are sub-allocated */
 113         unsigned offset;
 114
 115         union {
 116                 /* for _FD_RINGBUFFER_OBJECT case: */
 117                 struct {
 118                         struct fd_pipe *pipe;
 119                         DECLARE_ARRAY(struct msm_reloc_bo, reloc_bos);
 120                         struct set *ring_set;
 121                 };
 122                 /* for other cases: */
 123                 struct {
 124                         struct fd_submit *submit;
 125                         DECLARE_ARRAY(struct msm_cmd *, cmds);
 126                 };
 127         } u;
 128
 129         struct msm_cmd *cmd;          /* current cmd */
 130         struct fd_bo *ring_bo;
 131 };
 132 FD_DEFINE_CAST(fd_ringbuffer, msm_ringbuffer);
 133
 134 static void finalize_current_cmd(struct fd_ringbuffer *ring);
 135 static struct fd_ringbuffer * msm_ringbuffer_init(
 136                 struct msm_ringbuffer *msm_ring,
 137                 uint32_t size, enum fd_ringbuffer_flags flags);
 138
 139 /* add (if needed) bo to submit and return index: */
 140 static uint32_t
 141 append_bo(struct msm_submit *submit, struct fd_bo *bo, uint32_t flags)
 142 {
 143         struct msm_bo *msm_bo = to_msm_bo(bo);
 144         uint32_t idx;
 145
 146         /* NOTE: it is legal to use the same bo on different threads for
 147          * different submits.  But it is not legal to use the same submit
 148          * from given threads.
 149          */
 150         idx = READ_ONCE(msm_bo->idx);
 151
 152         if (unlikely((idx >= submit->nr_submit_bos) ||
 153                         (submit->submit_bos[idx].handle != bo->handle))) {
 154                 uint32_t hash = _mesa_hash_pointer(bo);
 155                 struct hash_entry *entry;
 156
 157                 entry = _mesa_hash_table_search_pre_hashed(submit->bo_table, hash, bo);
 158                 if (entry) {
 159                         /* found */
 160                         idx = (uint32_t)(uintptr_t)entry->data;
 161                 } else {
 162                         idx = APPEND(submit, submit_bos);
 163                         idx = APPEND(submit, bos);
 164
 165                         submit->submit_bos[idx].flags = 0;
 166                         submit->submit_bos[idx].handle = bo->handle;
 167                         submit->submit_bos[idx].presumed = 0;
 168
 169                         submit->bos[idx] = fd_bo_ref(bo);
 170
 171                         _mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo,
 172                                         (void *)(uintptr_t)idx);
 173                 }
 174                 msm_bo->idx = idx;
 175         }
 176
 177         if (flags & FD_RELOC_READ)
 178                 submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_READ;
 179         if (flags & FD_RELOC_WRITE)
 180                 submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
 181
 182         return idx;
 183 }
 184
 185 static void
 186 append_ring(struct set *set, struct fd_ringbuffer *ring)
 187 {
 188         uint32_t hash = _mesa_hash_pointer(ring);
 189
 190         if (!_mesa_set_search_pre_hashed(set, hash, ring)) {
 191                 fd_ringbuffer_ref(ring);
 192                 _mesa_set_add_pre_hashed(set, hash, ring);
 193         }
 194 }
 195
 196 static void
 197 msm_submit_suballoc_ring_bo(struct fd_submit *submit,
 198                 struct msm_ringbuffer *msm_ring, uint32_t size)
 199 {
 200         struct msm_submit *msm_submit = to_msm_submit(submit);
 201         unsigned suballoc_offset = 0;
 202         struct fd_bo *suballoc_bo = NULL;
 203
 204         if (msm_submit->suballoc_ring) {
 205                 struct msm_ringbuffer *suballoc_ring =
 206                                 to_msm_ringbuffer(msm_submit->suballoc_ring);
 207
 208                 suballoc_bo = suballoc_ring->ring_bo;
 209                 suballoc_offset = fd_ringbuffer_size(msm_submit->suballoc_ring) +
 210                                 suballoc_ring->offset;
 211
 212                 suballoc_offset = align(suballoc_offset, 0x10);
 213
 214                 if ((size + suballoc_offset) > suballoc_bo->size) {
 215                         suballoc_bo = NULL;
 216                 }
 217         }
 218
 219         if (!suballoc_bo) {
 220                 // TODO possibly larger size for streaming bo?
 221                 msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, 0x8000);
 222                 msm_ring->offset = 0;
 223         } else {
 224                 msm_ring->ring_bo = fd_bo_ref(suballoc_bo);
 225                 msm_ring->offset = suballoc_offset;
 226         }
 227
 228         struct fd_ringbuffer *old_suballoc_ring = msm_submit->suballoc_ring;
 229
 230         msm_submit->suballoc_ring = fd_ringbuffer_ref(&msm_ring->base);
 231
 232         if (old_suballoc_ring)
 233                 fd_ringbuffer_del(old_suballoc_ring);
 234 }
 235
 236 static struct fd_ringbuffer *
 237 msm_submit_new_ringbuffer(struct fd_submit *submit, uint32_t size,
 238                 enum fd_ringbuffer_flags flags)
 239 {
 240         struct msm_submit *msm_submit = to_msm_submit(submit);
 241         struct msm_ringbuffer *msm_ring;
 242
 243         msm_ring = slab_alloc_st(&msm_submit->ring_pool);
 244
 245         msm_ring->u.submit = submit;
 246
 247         /* NOTE: needs to be before _suballoc_ring_bo() since it could
 248          * increment the refcnt of the current ring
 249          */
 250         msm_ring->base.refcnt = 1;
 251
 252         if (flags & FD_RINGBUFFER_STREAMING) {
 253                 msm_submit_suballoc_ring_bo(submit, msm_ring, size);
 254         } else {
 255                 if (flags & FD_RINGBUFFER_GROWABLE)
 256                         size = INIT_SIZE;
 257
 258                 msm_ring->offset = 0;
 259                 msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, size);
 260         }
 261
 262         if (!msm_ringbuffer_init(msm_ring, size, flags))
 263                 return NULL;
 264
 265         if (flags & FD_RINGBUFFER_PRIMARY) {
 266                 debug_assert(!msm_submit->primary);
 267                 msm_submit->primary = fd_ringbuffer_ref(&msm_ring->base);
 268         }
 269
 270         return &msm_ring->base;
 271 }
 272
 273 static struct drm_msm_gem_submit_reloc *
 274 handle_stateobj_relocs(struct msm_submit *submit, struct msm_ringbuffer *ring)
 275 {
 276         struct msm_cmd *cmd = ring->cmd;
 277         struct drm_msm_gem_submit_reloc *relocs;
 278
 279         relocs = malloc(cmd->nr_relocs * sizeof(*relocs));
 280
 281         for (unsigned i = 0; i < cmd->nr_relocs; i++) {
 282                 unsigned idx = cmd->relocs[i].reloc_idx;
 283                 struct fd_bo *bo = ring->u.reloc_bos[idx].bo;
 284                 unsigned flags = 0;
 285
 286                 if (ring->u.reloc_bos[idx].flags & MSM_SUBMIT_BO_READ)
 287                         flags |= FD_RELOC_READ;
 288                 if (ring->u.reloc_bos[idx].flags & MSM_SUBMIT_BO_WRITE)
 289                         flags |= FD_RELOC_WRITE;
 290
 291                 relocs[i] = cmd->relocs[i];
 292                 relocs[i].reloc_idx = append_bo(submit, bo, flags);
 293         }
 294
 295         return relocs;
 296 }
 297
 298 static int
 299 msm_submit_flush(struct fd_submit *submit, int in_fence_fd,
 300                 int *out_fence_fd, uint32_t *out_fence)
 301 {
 302         struct msm_submit *msm_submit = to_msm_submit(submit);
 303         struct msm_pipe *msm_pipe = to_msm_pipe(submit->pipe);
 304         struct drm_msm_gem_submit req = {
 305                         .flags = msm_pipe->pipe,
 306                         .queueid = msm_pipe->queue_id,
 307         };
 308         int ret;
 309
 310         debug_assert(msm_submit->primary);
 311
 312         finalize_current_cmd(msm_submit->primary);
 313         append_ring(msm_submit->ring_set, msm_submit->primary);
 314
 315         unsigned nr_cmds = 0;
 316         unsigned nr_objs = 0;
 317
 318         set_foreach(msm_submit->ring_set, entry) {
 319                 struct fd_ringbuffer *ring = (void *)entry->key;
 320                 if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 321                         nr_cmds += 1;
 322                         nr_objs += 1;
 323                 } else {
 324                         if (ring != msm_submit->primary)
 325                                 finalize_current_cmd(ring);
 326                         nr_cmds += to_msm_ringbuffer(ring)->u.nr_cmds;
 327                 }
 328         }
 329
 330         void *obj_relocs[nr_objs];
 331         struct drm_msm_gem_submit_cmd cmds[nr_cmds];
 332         unsigned i = 0, o = 0;
 333
 334         set_foreach(msm_submit->ring_set, entry) {
 335                 struct fd_ringbuffer *ring = (void *)entry->key;
 336                 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 337
 338                 debug_assert(i < nr_cmds);
 339
 340                 // TODO handle relocs:
 341                 if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 342
 343                         debug_assert(o < nr_objs);
 344
 345                         void *relocs = handle_stateobj_relocs(msm_submit, msm_ring);
 346                         obj_relocs[o++] = relocs;
 347
 348                         cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
 349                         cmds[i].submit_idx =
 350                                 append_bo(msm_submit, msm_ring->ring_bo, FD_RELOC_READ);
 351                         cmds[i].submit_offset = msm_ring->offset;
 352                         cmds[i].size = offset_bytes(ring->cur, ring->start);
 353                         cmds[i].pad = 0;
 354                         cmds[i].nr_relocs = msm_ring->cmd->nr_relocs;
 355                         cmds[i].relocs = VOID2U64(relocs);
 356
 357                         i++;
 358                 } else {
 359                         for (unsigned j = 0; j < msm_ring->u.nr_cmds; j++) {
 360                                 if (ring->flags & FD_RINGBUFFER_PRIMARY) {
 361                                         cmds[i].type = MSM_SUBMIT_CMD_BUF;
 362                                 } else {
 363                                         cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
 364                                 }
 365                                 cmds[i].submit_idx = append_bo(msm_submit,
 366                                                 msm_ring->u.cmds[j]->ring_bo, FD_RELOC_READ);
 367                                 cmds[i].submit_offset = msm_ring->offset;
 368                                 cmds[i].size = msm_ring->u.cmds[j]->size;
 369                                 cmds[i].pad = 0;
 370                                 cmds[i].nr_relocs = msm_ring->u.cmds[j]->nr_relocs;
 371                                 cmds[i].relocs = VOID2U64(msm_ring->u.cmds[j]->relocs);
 372
 373                                 i++;
 374                         }
 375                 }
 376         }
 377
 378         if (in_fence_fd != -1) {
 379                 req.flags |= MSM_SUBMIT_FENCE_FD_IN | MSM_SUBMIT_NO_IMPLICIT;
 380                 req.fence_fd = in_fence_fd;
 381         }
 382
 383         if (out_fence_fd) {
 384                 req.flags |= MSM_SUBMIT_FENCE_FD_OUT;
 385         }
 386
 387         /* needs to be after get_cmd() as that could create bos/cmds table: */
 388         req.bos = VOID2U64(msm_submit->submit_bos),
 389         req.nr_bos = msm_submit->nr_submit_bos;
 390         req.cmds = VOID2U64(cmds),
 391         req.nr_cmds = nr_cmds;
 392
 393         DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos);
 394
 395         ret = drmCommandWriteRead(submit->pipe->dev->fd, DRM_MSM_GEM_SUBMIT,
 396                         &req, sizeof(req));
 397         if (ret) {
 398                 ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno));
 399                 msm_dump_submit(&req);
 400         } else if (!ret) {
 401                 if (out_fence)
 402                         *out_fence = req.fence;
 403
 404                 if (out_fence_fd)
 405                         *out_fence_fd = req.fence_fd;
 406         }
 407
 408         for (unsigned o = 0; o < nr_objs; o++)
 409                 free(obj_relocs[o]);
 410
 411         return ret;
 412 }
 413
 414 static void
 415 unref_rings(struct set_entry *entry)
 416 {
 417         struct fd_ringbuffer *ring = (void *)entry->key;
 418         fd_ringbuffer_del(ring);
 419 }
 420
 421 static void
 422 msm_submit_destroy(struct fd_submit *submit)
 423 {
 424         struct msm_submit *msm_submit = to_msm_submit(submit);
 425
 426         if (msm_submit->primary)
 427                 fd_ringbuffer_del(msm_submit->primary);
 428         if (msm_submit->suballoc_ring)
 429                 fd_ringbuffer_del(msm_submit->suballoc_ring);
 430
 431         _mesa_hash_table_destroy(msm_submit->bo_table, NULL);
 432         _mesa_set_destroy(msm_submit->ring_set, unref_rings);
 433
 434         // TODO it would be nice to have a way to debug_assert() if all
 435         // rb's haven't been free'd back to the slab, because that is
 436         // an indication that we are leaking bo's
 437         slab_destroy(&msm_submit->ring_pool);
 438
 439         for (unsigned i = 0; i < msm_submit->nr_bos; i++)
 440                 fd_bo_del(msm_submit->bos[i]);
 441
 442         free(msm_submit->submit_bos);
 443         free(msm_submit->bos);
 444         free(msm_submit);
 445 }
 446
 447 static const struct fd_submit_funcs submit_funcs = {
 448                 .new_ringbuffer = msm_submit_new_ringbuffer,
 449                 .flush = msm_submit_flush,
 450                 .destroy = msm_submit_destroy,
 451 };
 452
 453 struct fd_submit *
 454 msm_submit_new(struct fd_pipe *pipe)
 455 {
 456         struct msm_submit *msm_submit = calloc(1, sizeof(*msm_submit));
 457         struct fd_submit *submit;
 458
 459         msm_submit->bo_table = _mesa_hash_table_create(NULL,
 460                         _mesa_hash_pointer, _mesa_key_pointer_equal);
 461         msm_submit->ring_set = _mesa_set_create(NULL,
 462                         _mesa_hash_pointer, _mesa_key_pointer_equal);
 463         // TODO tune size:
 464         slab_create(&msm_submit->ring_pool, sizeof(struct msm_ringbuffer), 16);
 465
 466         submit = &msm_submit->base;
 467         submit->pipe = pipe;
 468         submit->funcs = &submit_funcs;
 469
 470         return submit;
 471 }
 472
 473
 474 static void
 475 finalize_current_cmd(struct fd_ringbuffer *ring)
 476 {
 477         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 478
 479         debug_assert(!(ring->flags & _FD_RINGBUFFER_OBJECT));
 480
 481         if (!msm_ring->cmd)
 482                 return;
 483
 484         debug_assert(msm_ring->cmd->ring_bo == msm_ring->ring_bo);
 485
 486         unsigned idx = APPEND(&msm_ring->u, cmds);
 487
 488         msm_ring->u.cmds[idx] = msm_ring->cmd;
 489         msm_ring->cmd = NULL;
 490
 491         msm_ring->u.cmds[idx]->size = offset_bytes(ring->cur, ring->start);
 492 }
 493
 494 static void
 495 msm_ringbuffer_grow(struct fd_ringbuffer *ring, uint32_t size)
 496 {
 497         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 498         struct fd_pipe *pipe = msm_ring->u.submit->pipe;
 499
 500         debug_assert(ring->flags & FD_RINGBUFFER_GROWABLE);
 501
 502         finalize_current_cmd(ring);
 503
 504         fd_bo_del(msm_ring->ring_bo);
 505         msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size);
 506         msm_ring->cmd = cmd_new(msm_ring->ring_bo);
 507
 508         ring->start = fd_bo_map(msm_ring->ring_bo);
 509         ring->end = &(ring->start[size/4]);
 510         ring->cur = ring->start;
 511         ring->size = size;
 512 }
 513
 514 static void
 515 msm_ringbuffer_emit_reloc(struct fd_ringbuffer *ring,
 516                 const struct fd_reloc *reloc)
 517 {
 518         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 519         struct fd_pipe *pipe;
 520         unsigned reloc_idx;
 521
 522         if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 523                 unsigned idx = APPEND(&msm_ring->u, reloc_bos);
 524
 525                 msm_ring->u.reloc_bos[idx].bo = fd_bo_ref(reloc->bo);
 526                 msm_ring->u.reloc_bos[idx].flags = reloc->flags;
 527
 528                 /* this gets fixed up at submit->flush() time, since this state-
 529                  * object rb can be used with many different submits
 530                  */
 531                 reloc_idx = idx;
 532
 533                 pipe = msm_ring->u.pipe;
 534         } else {
 535                 struct msm_submit *msm_submit =
 536                                 to_msm_submit(msm_ring->u.submit);
 537
 538                 reloc_idx = append_bo(msm_submit, reloc->bo, reloc->flags);
 539
 540                 pipe = msm_ring->u.submit->pipe;
 541         }
 542
 543         struct drm_msm_gem_submit_reloc *r;
 544         unsigned idx = APPEND(msm_ring->cmd, relocs);
 545
 546         r = &msm_ring->cmd->relocs[idx];
 547
 548         r->reloc_idx = reloc_idx;
 549         r->reloc_offset = reloc->offset;
 550         r->or = reloc->or;
 551         r->shift = reloc->shift;
 552         r->submit_offset = offset_bytes(ring->cur, ring->start) +
 553                         msm_ring->offset;
 554
 555         ring->cur++;
 556
 557         if (pipe->gpu_id >= 500) {
 558                 idx = APPEND(msm_ring->cmd, relocs);
 559                 r = &msm_ring->cmd->relocs[idx];
 560
 561                 r->reloc_idx = reloc_idx;
 562                 r->reloc_offset = reloc->offset;
 563                 r->or = reloc->orhi;
 564                 r->shift = reloc->shift - 32;
 565                 r->submit_offset = offset_bytes(ring->cur, ring->start) +
 566                                 msm_ring->offset;
 567
 568                 ring->cur++;
 569         }
 570 }
 571
 572 static void
 573 append_stateobj_rings(struct msm_submit *submit, struct fd_ringbuffer *target)
 574 {
 575         struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target);
 576
 577         debug_assert(target->flags & _FD_RINGBUFFER_OBJECT);
 578
 579         set_foreach(msm_target->u.ring_set, entry) {
 580                 struct fd_ringbuffer *ring = (void *)entry->key;
 581
 582                 append_ring(submit->ring_set, ring);
 583
 584                 if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 585                         append_stateobj_rings(submit, ring);
 586                 }
 587         }
 588 }
 589
 590 static uint32_t
 591 msm_ringbuffer_emit_reloc_ring(struct fd_ringbuffer *ring,
 592                 struct fd_ringbuffer *target, uint32_t cmd_idx)
 593 {
 594         struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target);
 595         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 596         struct fd_bo *bo;
 597         uint32_t size;
 598
 599         if ((target->flags & FD_RINGBUFFER_GROWABLE) &&
 600                         (cmd_idx < msm_target->u.nr_cmds)) {
 601                 bo   = msm_target->u.cmds[cmd_idx]->ring_bo;
 602                 size = msm_target->u.cmds[cmd_idx]->size;
 603         } else {
 604                 bo   = msm_target->ring_bo;
 605                 size = offset_bytes(target->cur, target->start);
 606         }
 607
 608         msm_ringbuffer_emit_reloc(ring, &(struct fd_reloc){
 609                 .bo     = bo,
 610                 .flags  = FD_RELOC_READ,
 611                 .offset = msm_target->offset,
 612         });
 613
 614         if (!size)
 615                 return 0;
 616
 617         if ((target->flags & _FD_RINGBUFFER_OBJECT) &&
 618                         !(ring->flags & _FD_RINGBUFFER_OBJECT)) {
 619                 struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
 620
 621                 append_stateobj_rings(msm_submit, target);
 622         }
 623
 624         if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 625                 append_ring(msm_ring->u.ring_set, target);
 626         } else {
 627                 struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
 628                 append_ring(msm_submit->ring_set, target);
 629         }
 630
 631         return size;
 632 }
 633
 634 static uint32_t
 635 msm_ringbuffer_cmd_count(struct fd_ringbuffer *ring)
 636 {
 637         if (ring->flags & FD_RINGBUFFER_GROWABLE)
 638                 return to_msm_ringbuffer(ring)->u.nr_cmds + 1;
 639         return 1;
 640 }
 641
 642 static void
 643 msm_ringbuffer_destroy(struct fd_ringbuffer *ring)
 644 {
 645         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 646
 647         fd_bo_del(msm_ring->ring_bo);
 648         if (msm_ring->cmd)
 649                 cmd_free(msm_ring->cmd);
 650
 651         if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 652                 for (unsigned i = 0; i < msm_ring->u.nr_reloc_bos; i++) {
 653                         fd_bo_del(msm_ring->u.reloc_bos[i].bo);
 654                 }
 655
 656                 _mesa_set_destroy(msm_ring->u.ring_set, unref_rings);
 657
 658                 free(msm_ring->u.reloc_bos);
 659                 free(msm_ring);
 660         } else {
 661                 struct fd_submit *submit = msm_ring->u.submit;
 662
 663                 for (unsigned i = 0; i < msm_ring->u.nr_cmds; i++) {
 664                         cmd_free(msm_ring->u.cmds[i]);
 665                 }
 666
 667                 free(msm_ring->u.cmds);
 668                 slab_free_st(&to_msm_submit(submit)->ring_pool, msm_ring);
 669         }
 670 }
 671
 672 static const struct fd_ringbuffer_funcs ring_funcs = {
 673                 .grow = msm_ringbuffer_grow,
 674                 .emit_reloc = msm_ringbuffer_emit_reloc,
 675                 .emit_reloc_ring = msm_ringbuffer_emit_reloc_ring,
 676                 .cmd_count = msm_ringbuffer_cmd_count,
 677                 .destroy = msm_ringbuffer_destroy,
 678 };
 679
 680 static inline struct fd_ringbuffer *
 681 msm_ringbuffer_init(struct msm_ringbuffer *msm_ring, uint32_t size,
 682                 enum fd_ringbuffer_flags flags)
 683 {
 684         struct fd_ringbuffer *ring = &msm_ring->base;
 685
 686         debug_assert(msm_ring->ring_bo);
 687
 688         uint8_t *base = fd_bo_map(msm_ring->ring_bo);
 689         ring->start = (void *)(base + msm_ring->offset);
 690         ring->end = &(ring->start[size/4]);
 691         ring->cur = ring->start;
 692
 693         ring->size = size;
 694         ring->flags = flags;
 695
 696         ring->funcs = &ring_funcs;
 697
 698         msm_ring->u.cmds = NULL;
 699         msm_ring->u.nr_cmds = msm_ring->u.max_cmds = 0;
 700
 701         msm_ring->cmd = cmd_new(msm_ring->ring_bo);
 702
 703         return ring;
 704 }
 705
 706 struct fd_ringbuffer *
 707 msm_ringbuffer_new_object(struct fd_pipe *pipe, uint32_t size)
 708 {
 709         struct msm_ringbuffer *msm_ring = malloc(sizeof(*msm_ring));
 710
 711         msm_ring->u.pipe = pipe;
 712         msm_ring->offset = 0;
 713         msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size);
 714         msm_ring->base.refcnt = 1;
 715
 716         msm_ring->u.reloc_bos = NULL;
 717         msm_ring->u.nr_reloc_bos = msm_ring->u.max_reloc_bos = 0;
 718
 719         msm_ring->u.ring_set = _mesa_set_create(NULL,
 720                         _mesa_hash_pointer, _mesa_key_pointer_equal);
 721
 722         return msm_ringbuffer_init(msm_ring, size, _FD_RINGBUFFER_OBJECT);
 723 }