src/freedreno/drm/msm_ringbuffer.c

   1 /*
   2  * Copyright (C) 2012-2018 Rob Clark <robclark@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark <robclark@freedesktop.org>
  25  */
  26
  27 #include <assert.h>
  28 #include <inttypes.h>
  29
  30 #include "util/hash_table.h"
  31 #include "util/set.h"
  32 #include "util/slab.h"
  33
  34 #include "drm/freedreno_ringbuffer.h"
  35 #include "msm_priv.h"
  36
  37 /* The legacy implementation of submit/ringbuffer, which still does the
  38  * traditional reloc and cmd tracking
  39  */
  40
  41
  42 #define INIT_SIZE 0x1000
  43
  44 static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER;
  45
  46
  47 struct msm_submit {
  48         struct fd_submit base;
  49
  50         DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos);
  51         DECLARE_ARRAY(struct fd_bo *, bos);
  52
  53         unsigned seqno;
  54
  55         /* maps fd_bo to idx in bos table: */
  56         struct hash_table *bo_table;
  57
  58         struct slab_mempool ring_pool;
  59
  60         /* hash-set of associated rings: */
  61         struct set *ring_set;
  62
  63         struct fd_ringbuffer *primary;
  64
  65         /* Allow for sub-allocation of stateobj ring buffers (ie. sharing
  66          * the same underlying bo)..
  67          *
  68          * We also rely on previous stateobj having been fully constructed
  69          * so we can reclaim extra space at it's end.
  70          */
  71         struct fd_ringbuffer *suballoc_ring;
  72 };
  73 FD_DEFINE_CAST(fd_submit, msm_submit);
  74
  75 /* for FD_RINGBUFFER_GROWABLE rb's, tracks the 'finalized' cmdstream buffers
  76  * and sizes.  Ie. a finalized buffer can have no more commands appended to
  77  * it.
  78  */
  79 struct msm_cmd {
  80         struct fd_bo *ring_bo;
  81         unsigned size;
  82         DECLARE_ARRAY(struct drm_msm_gem_submit_reloc, relocs);
  83 };
  84
  85 static struct msm_cmd *
  86 cmd_new(struct fd_bo *ring_bo)
  87 {
  88         struct msm_cmd *cmd = malloc(sizeof(*cmd));
  89         cmd->ring_bo = fd_bo_ref(ring_bo);
  90         cmd->size = 0;
  91         cmd->nr_relocs = cmd->max_relocs = 0;
  92         cmd->relocs = NULL;
  93         return cmd;
  94 }
  95
  96 static void
  97 cmd_free(struct msm_cmd *cmd)
  98 {
  99         fd_bo_del(cmd->ring_bo);
 100         free(cmd);
 101 }
 102
 103 /* for _FD_RINGBUFFER_OBJECT rb's we need to track the bo's and flags to
 104  * later copy into the submit when the stateobj rb is later referenced by
 105  * a regular rb:
 106  */
 107 struct msm_reloc_bo {
 108         struct fd_bo *bo;
 109         unsigned flags;
 110 };
 111
 112 struct msm_ringbuffer {
 113         struct fd_ringbuffer base;
 114
 115         /* for FD_RINGBUFFER_STREAMING rb's which are sub-allocated */
 116         unsigned offset;
 117
 118         union {
 119                 /* for _FD_RINGBUFFER_OBJECT case: */
 120                 struct {
 121                         struct fd_pipe *pipe;
 122                         DECLARE_ARRAY(struct msm_reloc_bo, reloc_bos);
 123                         struct set *ring_set;
 124                 };
 125                 /* for other cases: */
 126                 struct {
 127                         struct fd_submit *submit;
 128                         DECLARE_ARRAY(struct msm_cmd *, cmds);
 129                 };
 130         } u;
 131
 132         struct msm_cmd *cmd;          /* current cmd */
 133         struct fd_bo *ring_bo;
 134 };
 135 FD_DEFINE_CAST(fd_ringbuffer, msm_ringbuffer);
 136
 137 static void finalize_current_cmd(struct fd_ringbuffer *ring);
 138 static struct fd_ringbuffer * msm_ringbuffer_init(
 139                 struct msm_ringbuffer *msm_ring,
 140                 uint32_t size, enum fd_ringbuffer_flags flags);
 141
 142 /* add (if needed) bo to submit and return index: */
 143 static uint32_t
 144 append_bo(struct msm_submit *submit, struct fd_bo *bo, uint32_t flags)
 145 {
 146         struct msm_bo *msm_bo = to_msm_bo(bo);
 147         uint32_t idx;
 148         pthread_mutex_lock(&idx_lock);
 149         if (likely(msm_bo->current_submit_seqno == submit->seqno)) {
 150                 idx = msm_bo->idx;
 151         } else {
 152                 uint32_t hash = _mesa_hash_pointer(bo);
 153                 struct hash_entry *entry;
 154
 155                 entry = _mesa_hash_table_search_pre_hashed(submit->bo_table, hash, bo);
 156                 if (entry) {
 157                         /* found */
 158                         idx = (uint32_t)(uintptr_t)entry->data;
 159                 } else {
 160                         idx = APPEND(submit, submit_bos);
 161                         idx = APPEND(submit, bos);
 162
 163                         submit->submit_bos[idx].flags = 0;
 164                         submit->submit_bos[idx].handle = bo->handle;
 165                         submit->submit_bos[idx].presumed = 0;
 166
 167                         submit->bos[idx] = fd_bo_ref(bo);
 168
 169                         _mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo,
 170                                         (void *)(uintptr_t)idx);
 171                 }
 172                 msm_bo->current_submit_seqno = submit->seqno;
 173                 msm_bo->idx = idx;
 174         }
 175         pthread_mutex_unlock(&idx_lock);
 176         if (flags & FD_RELOC_READ)
 177                 submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_READ;
 178         if (flags & FD_RELOC_WRITE)
 179                 submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
 180         return idx;
 181 }
 182
 183 static void
 184 append_ring(struct set *set, struct fd_ringbuffer *ring)
 185 {
 186         uint32_t hash = _mesa_hash_pointer(ring);
 187
 188         if (!_mesa_set_search_pre_hashed(set, hash, ring)) {
 189                 fd_ringbuffer_ref(ring);
 190                 _mesa_set_add_pre_hashed(set, hash, ring);
 191         }
 192 }
 193
 194 static void
 195 msm_submit_suballoc_ring_bo(struct fd_submit *submit,
 196                 struct msm_ringbuffer *msm_ring, uint32_t size)
 197 {
 198         struct msm_submit *msm_submit = to_msm_submit(submit);
 199         unsigned suballoc_offset = 0;
 200         struct fd_bo *suballoc_bo = NULL;
 201
 202         if (msm_submit->suballoc_ring) {
 203                 struct msm_ringbuffer *suballoc_ring =
 204                                 to_msm_ringbuffer(msm_submit->suballoc_ring);
 205
 206                 suballoc_bo = suballoc_ring->ring_bo;
 207                 suballoc_offset = fd_ringbuffer_size(msm_submit->suballoc_ring) +
 208                                 suballoc_ring->offset;
 209
 210                 suballoc_offset = align(suballoc_offset, 0x10);
 211
 212                 if ((size + suballoc_offset) > suballoc_bo->size) {
 213                         suballoc_bo = NULL;
 214                 }
 215         }
 216
 217         if (!suballoc_bo) {
 218                 // TODO possibly larger size for streaming bo?
 219                 msm_ring->ring_bo = fd_bo_new_ring(
 220                                 submit->pipe->dev, 0x8000, 0);
 221                 msm_ring->offset = 0;
 222         } else {
 223                 msm_ring->ring_bo = fd_bo_ref(suballoc_bo);
 224                 msm_ring->offset = suballoc_offset;
 225         }
 226
 227         struct fd_ringbuffer *old_suballoc_ring = msm_submit->suballoc_ring;
 228
 229         msm_submit->suballoc_ring = fd_ringbuffer_ref(&msm_ring->base);
 230
 231         if (old_suballoc_ring)
 232                 fd_ringbuffer_del(old_suballoc_ring);
 233 }
 234
 235 static struct fd_ringbuffer *
 236 msm_submit_new_ringbuffer(struct fd_submit *submit, uint32_t size,
 237                 enum fd_ringbuffer_flags flags)
 238 {
 239         struct msm_submit *msm_submit = to_msm_submit(submit);
 240         struct msm_ringbuffer *msm_ring;
 241
 242         msm_ring = slab_alloc_st(&msm_submit->ring_pool);
 243
 244         msm_ring->u.submit = submit;
 245
 246         /* NOTE: needs to be before _suballoc_ring_bo() since it could
 247          * increment the refcnt of the current ring
 248          */
 249         msm_ring->base.refcnt = 1;
 250
 251         if (flags & FD_RINGBUFFER_STREAMING) {
 252                 msm_submit_suballoc_ring_bo(submit, msm_ring, size);
 253         } else {
 254                 if (flags & FD_RINGBUFFER_GROWABLE)
 255                         size = INIT_SIZE;
 256
 257                 msm_ring->offset = 0;
 258                 msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, size, 0);
 259         }
 260
 261         if (!msm_ringbuffer_init(msm_ring, size, flags))
 262                 return NULL;
 263
 264         if (flags & FD_RINGBUFFER_PRIMARY) {
 265                 debug_assert(!msm_submit->primary);
 266                 msm_submit->primary = fd_ringbuffer_ref(&msm_ring->base);
 267         }
 268
 269         return &msm_ring->base;
 270 }
 271
 272 static struct drm_msm_gem_submit_reloc *
 273 handle_stateobj_relocs(struct msm_submit *submit, struct msm_ringbuffer *ring)
 274 {
 275         struct msm_cmd *cmd = ring->cmd;
 276         struct drm_msm_gem_submit_reloc *relocs;
 277
 278         relocs = malloc(cmd->nr_relocs * sizeof(*relocs));
 279
 280         for (unsigned i = 0; i < cmd->nr_relocs; i++) {
 281                 unsigned idx = cmd->relocs[i].reloc_idx;
 282                 struct fd_bo *bo = ring->u.reloc_bos[idx].bo;
 283                 unsigned flags = 0;
 284
 285                 if (ring->u.reloc_bos[idx].flags & MSM_SUBMIT_BO_READ)
 286                         flags |= FD_RELOC_READ;
 287                 if (ring->u.reloc_bos[idx].flags & MSM_SUBMIT_BO_WRITE)
 288                         flags |= FD_RELOC_WRITE;
 289
 290                 relocs[i] = cmd->relocs[i];
 291                 relocs[i].reloc_idx = append_bo(submit, bo, flags);
 292         }
 293
 294         return relocs;
 295 }
 296
 297 static int
 298 msm_submit_flush(struct fd_submit *submit, int in_fence_fd,
 299                 int *out_fence_fd, uint32_t *out_fence)
 300 {
 301         struct msm_submit *msm_submit = to_msm_submit(submit);
 302         struct msm_pipe *msm_pipe = to_msm_pipe(submit->pipe);
 303         struct drm_msm_gem_submit req = {
 304                         .flags = msm_pipe->pipe,
 305                         .queueid = msm_pipe->queue_id,
 306         };
 307         int ret;
 308
 309         debug_assert(msm_submit->primary);
 310
 311         finalize_current_cmd(msm_submit->primary);
 312         append_ring(msm_submit->ring_set, msm_submit->primary);
 313
 314         unsigned nr_cmds = 0;
 315         unsigned nr_objs = 0;
 316
 317         set_foreach(msm_submit->ring_set, entry) {
 318                 struct fd_ringbuffer *ring = (void *)entry->key;
 319                 if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 320                         nr_cmds += 1;
 321                         nr_objs += 1;
 322                 } else {
 323                         if (ring != msm_submit->primary)
 324                                 finalize_current_cmd(ring);
 325                         nr_cmds += to_msm_ringbuffer(ring)->u.nr_cmds;
 326                 }
 327         }
 328
 329         void *obj_relocs[nr_objs];
 330         struct drm_msm_gem_submit_cmd cmds[nr_cmds];
 331         unsigned i = 0, o = 0;
 332
 333         set_foreach(msm_submit->ring_set, entry) {
 334                 struct fd_ringbuffer *ring = (void *)entry->key;
 335                 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 336
 337                 debug_assert(i < nr_cmds);
 338
 339                 // TODO handle relocs:
 340                 if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 341
 342                         debug_assert(o < nr_objs);
 343
 344                         void *relocs = handle_stateobj_relocs(msm_submit, msm_ring);
 345                         obj_relocs[o++] = relocs;
 346
 347                         cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
 348                         cmds[i].submit_idx =
 349                                 append_bo(msm_submit, msm_ring->ring_bo, FD_RELOC_READ);
 350                         cmds[i].submit_offset = msm_ring->offset;
 351                         cmds[i].size = offset_bytes(ring->cur, ring->start);
 352                         cmds[i].pad = 0;
 353                         cmds[i].nr_relocs = msm_ring->cmd->nr_relocs;
 354                         cmds[i].relocs = VOID2U64(relocs);
 355
 356                         i++;
 357                 } else {
 358                         for (unsigned j = 0; j < msm_ring->u.nr_cmds; j++) {
 359                                 if (ring->flags & FD_RINGBUFFER_PRIMARY) {
 360                                         cmds[i].type = MSM_SUBMIT_CMD_BUF;
 361                                 } else {
 362                                         cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
 363                                 }
 364                                 cmds[i].submit_idx = append_bo(msm_submit,
 365                                                 msm_ring->u.cmds[j]->ring_bo, FD_RELOC_READ);
 366                                 cmds[i].submit_offset = msm_ring->offset;
 367                                 cmds[i].size = msm_ring->u.cmds[j]->size;
 368                                 cmds[i].pad = 0;
 369                                 cmds[i].nr_relocs = msm_ring->u.cmds[j]->nr_relocs;
 370                                 cmds[i].relocs = VOID2U64(msm_ring->u.cmds[j]->relocs);
 371
 372                                 i++;
 373                         }
 374                 }
 375         }
 376
 377         if (in_fence_fd != -1) {
 378                 req.flags |= MSM_SUBMIT_FENCE_FD_IN | MSM_SUBMIT_NO_IMPLICIT;
 379                 req.fence_fd = in_fence_fd;
 380         }
 381
 382         if (out_fence_fd) {
 383                 req.flags |= MSM_SUBMIT_FENCE_FD_OUT;
 384         }
 385
 386         /* needs to be after get_cmd() as that could create bos/cmds table: */
 387         req.bos = VOID2U64(msm_submit->submit_bos),
 388         req.nr_bos = msm_submit->nr_submit_bos;
 389         req.cmds = VOID2U64(cmds),
 390         req.nr_cmds = nr_cmds;
 391
 392         DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos);
 393
 394         ret = drmCommandWriteRead(submit->pipe->dev->fd, DRM_MSM_GEM_SUBMIT,
 395                         &req, sizeof(req));
 396         if (ret) {
 397                 ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno));
 398                 msm_dump_submit(&req);
 399         } else if (!ret) {
 400                 if (out_fence)
 401                         *out_fence = req.fence;
 402
 403                 if (out_fence_fd)
 404                         *out_fence_fd = req.fence_fd;
 405         }
 406
 407         for (unsigned o = 0; o < nr_objs; o++)
 408                 free(obj_relocs[o]);
 409
 410         return ret;
 411 }
 412
 413 static void
 414 unref_rings(struct set_entry *entry)
 415 {
 416         struct fd_ringbuffer *ring = (void *)entry->key;
 417         fd_ringbuffer_del(ring);
 418 }
 419
 420 static void
 421 msm_submit_destroy(struct fd_submit *submit)
 422 {
 423         struct msm_submit *msm_submit = to_msm_submit(submit);
 424
 425         if (msm_submit->primary)
 426                 fd_ringbuffer_del(msm_submit->primary);
 427         if (msm_submit->suballoc_ring)
 428                 fd_ringbuffer_del(msm_submit->suballoc_ring);
 429
 430         _mesa_hash_table_destroy(msm_submit->bo_table, NULL);
 431         _mesa_set_destroy(msm_submit->ring_set, unref_rings);
 432
 433         // TODO it would be nice to have a way to debug_assert() if all
 434         // rb's haven't been free'd back to the slab, because that is
 435         // an indication that we are leaking bo's
 436         slab_destroy(&msm_submit->ring_pool);
 437
 438         for (unsigned i = 0; i < msm_submit->nr_bos; i++)
 439                 fd_bo_del(msm_submit->bos[i]);
 440
 441         free(msm_submit->submit_bos);
 442         free(msm_submit->bos);
 443         free(msm_submit);
 444 }
 445
 446 static const struct fd_submit_funcs submit_funcs = {
 447                 .new_ringbuffer = msm_submit_new_ringbuffer,
 448                 .flush = msm_submit_flush,
 449                 .destroy = msm_submit_destroy,
 450 };
 451
 452 struct fd_submit *
 453 msm_submit_new(struct fd_pipe *pipe)
 454 {
 455         struct msm_submit *msm_submit = calloc(1, sizeof(*msm_submit));
 456         struct fd_submit *submit;
 457         static unsigned submit_cnt = 0;
 458
 459         msm_submit->seqno = ++submit_cnt;
 460         msm_submit->bo_table = _mesa_hash_table_create(NULL,
 461                         _mesa_hash_pointer, _mesa_key_pointer_equal);
 462         msm_submit->ring_set = _mesa_set_create(NULL,
 463                         _mesa_hash_pointer, _mesa_key_pointer_equal);
 464         // TODO tune size:
 465         slab_create(&msm_submit->ring_pool, sizeof(struct msm_ringbuffer), 16);
 466
 467         submit = &msm_submit->base;
 468         submit->pipe = pipe;
 469         submit->funcs = &submit_funcs;
 470
 471         return submit;
 472 }
 473
 474
 475 static void
 476 finalize_current_cmd(struct fd_ringbuffer *ring)
 477 {
 478         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 479
 480         debug_assert(!(ring->flags & _FD_RINGBUFFER_OBJECT));
 481
 482         if (!msm_ring->cmd)
 483                 return;
 484
 485         debug_assert(msm_ring->cmd->ring_bo == msm_ring->ring_bo);
 486
 487         unsigned idx = APPEND(&msm_ring->u, cmds);
 488
 489         msm_ring->u.cmds[idx] = msm_ring->cmd;
 490         msm_ring->cmd = NULL;
 491
 492         msm_ring->u.cmds[idx]->size = offset_bytes(ring->cur, ring->start);
 493 }
 494
 495 static void
 496 msm_ringbuffer_grow(struct fd_ringbuffer *ring, uint32_t size)
 497 {
 498         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 499         struct fd_pipe *pipe = msm_ring->u.submit->pipe;
 500
 501         debug_assert(ring->flags & FD_RINGBUFFER_GROWABLE);
 502
 503         finalize_current_cmd(ring);
 504
 505         fd_bo_del(msm_ring->ring_bo);
 506         msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size, 0);
 507         msm_ring->cmd = cmd_new(msm_ring->ring_bo);
 508
 509         ring->start = fd_bo_map(msm_ring->ring_bo);
 510         ring->end = &(ring->start[size/4]);
 511         ring->cur = ring->start;
 512         ring->size = size;
 513 }
 514
 515 static void
 516 msm_ringbuffer_emit_reloc(struct fd_ringbuffer *ring,
 517                 const struct fd_reloc *reloc)
 518 {
 519         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 520         struct fd_pipe *pipe;
 521         unsigned reloc_idx;
 522
 523         if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 524                 unsigned idx = APPEND(&msm_ring->u, reloc_bos);
 525
 526                 msm_ring->u.reloc_bos[idx].bo = fd_bo_ref(reloc->bo);
 527                 msm_ring->u.reloc_bos[idx].flags = reloc->flags;
 528
 529                 /* this gets fixed up at submit->flush() time, since this state-
 530                  * object rb can be used with many different submits
 531                  */
 532                 reloc_idx = idx;
 533
 534                 pipe = msm_ring->u.pipe;
 535         } else {
 536                 struct msm_submit *msm_submit =
 537                                 to_msm_submit(msm_ring->u.submit);
 538
 539                 reloc_idx = append_bo(msm_submit, reloc->bo, reloc->flags);
 540
 541                 pipe = msm_ring->u.submit->pipe;
 542         }
 543
 544         struct drm_msm_gem_submit_reloc *r;
 545         unsigned idx = APPEND(msm_ring->cmd, relocs);
 546
 547         r = &msm_ring->cmd->relocs[idx];
 548
 549         r->reloc_idx = reloc_idx;
 550         r->reloc_offset = reloc->offset;
 551         r->or = reloc->or;
 552         r->shift = reloc->shift;
 553         r->submit_offset = offset_bytes(ring->cur, ring->start) +
 554                         msm_ring->offset;
 555
 556         ring->cur++;
 557
 558         if (pipe->gpu_id >= 500) {
 559                 idx = APPEND(msm_ring->cmd, relocs);
 560                 r = &msm_ring->cmd->relocs[idx];
 561
 562                 r->reloc_idx = reloc_idx;
 563                 r->reloc_offset = reloc->offset;
 564                 r->or = reloc->orhi;
 565                 r->shift = reloc->shift - 32;
 566                 r->submit_offset = offset_bytes(ring->cur, ring->start) +
 567                                 msm_ring->offset;
 568
 569                 ring->cur++;
 570         }
 571 }
 572
 573 static void
 574 append_stateobj_rings(struct msm_submit *submit, struct fd_ringbuffer *target)
 575 {
 576         struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target);
 577
 578         debug_assert(target->flags & _FD_RINGBUFFER_OBJECT);
 579
 580         set_foreach(msm_target->u.ring_set, entry) {
 581                 struct fd_ringbuffer *ring = (void *)entry->key;
 582
 583                 append_ring(submit->ring_set, ring);
 584
 585                 if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 586                         append_stateobj_rings(submit, ring);
 587                 }
 588         }
 589 }
 590
 591 static uint32_t
 592 msm_ringbuffer_emit_reloc_ring(struct fd_ringbuffer *ring,
 593                 struct fd_ringbuffer *target, uint32_t cmd_idx)
 594 {
 595         struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target);
 596         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 597         struct fd_bo *bo;
 598         uint32_t size;
 599
 600         if ((target->flags & FD_RINGBUFFER_GROWABLE) &&
 601                         (cmd_idx < msm_target->u.nr_cmds)) {
 602                 bo   = msm_target->u.cmds[cmd_idx]->ring_bo;
 603                 size = msm_target->u.cmds[cmd_idx]->size;
 604         } else {
 605                 bo   = msm_target->ring_bo;
 606                 size = offset_bytes(target->cur, target->start);
 607         }
 608
 609         msm_ringbuffer_emit_reloc(ring, &(struct fd_reloc){
 610                 .bo     = bo,
 611                 .flags  = FD_RELOC_READ,
 612                 .offset = msm_target->offset,
 613         });
 614
 615         if ((target->flags & _FD_RINGBUFFER_OBJECT) &&
 616                         !(ring->flags & _FD_RINGBUFFER_OBJECT)) {
 617                 struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
 618
 619                 append_stateobj_rings(msm_submit, target);
 620         }
 621
 622         if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 623                 append_ring(msm_ring->u.ring_set, target);
 624         } else {
 625                 struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
 626                 append_ring(msm_submit->ring_set, target);
 627         }
 628
 629         return size;
 630 }
 631
 632 static uint32_t
 633 msm_ringbuffer_cmd_count(struct fd_ringbuffer *ring)
 634 {
 635         if (ring->flags & FD_RINGBUFFER_GROWABLE)
 636                 return to_msm_ringbuffer(ring)->u.nr_cmds + 1;
 637         return 1;
 638 }
 639
 640 static void
 641 msm_ringbuffer_destroy(struct fd_ringbuffer *ring)
 642 {
 643         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 644
 645         fd_bo_del(msm_ring->ring_bo);
 646         if (msm_ring->cmd)
 647                 cmd_free(msm_ring->cmd);
 648
 649         if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 650                 for (unsigned i = 0; i < msm_ring->u.nr_reloc_bos; i++) {
 651                         fd_bo_del(msm_ring->u.reloc_bos[i].bo);
 652                 }
 653
 654                 _mesa_set_destroy(msm_ring->u.ring_set, unref_rings);
 655
 656                 free(msm_ring);
 657         } else {
 658                 struct fd_submit *submit = msm_ring->u.submit;
 659
 660                 for (unsigned i = 0; i < msm_ring->u.nr_cmds; i++) {
 661                         cmd_free(msm_ring->u.cmds[i]);
 662                 }
 663
 664                 slab_free_st(&to_msm_submit(submit)->ring_pool, msm_ring);
 665         }
 666 }
 667
 668 static const struct fd_ringbuffer_funcs ring_funcs = {
 669                 .grow = msm_ringbuffer_grow,
 670                 .emit_reloc = msm_ringbuffer_emit_reloc,
 671                 .emit_reloc_ring = msm_ringbuffer_emit_reloc_ring,
 672                 .cmd_count = msm_ringbuffer_cmd_count,
 673                 .destroy = msm_ringbuffer_destroy,
 674 };
 675
 676 static inline struct fd_ringbuffer *
 677 msm_ringbuffer_init(struct msm_ringbuffer *msm_ring, uint32_t size,
 678                 enum fd_ringbuffer_flags flags)
 679 {
 680         struct fd_ringbuffer *ring = &msm_ring->base;
 681
 682         debug_assert(msm_ring->ring_bo);
 683
 684         uint8_t *base = fd_bo_map(msm_ring->ring_bo);
 685         ring->start = (void *)(base + msm_ring->offset);
 686         ring->end = &(ring->start[size/4]);
 687         ring->cur = ring->start;
 688
 689         ring->size = size;
 690         ring->flags = flags;
 691
 692         ring->funcs = &ring_funcs;
 693
 694         msm_ring->u.cmds = NULL;
 695         msm_ring->u.nr_cmds = msm_ring->u.max_cmds = 0;
 696
 697         msm_ring->cmd = cmd_new(msm_ring->ring_bo);
 698
 699         return ring;
 700 }
 701
 702 struct fd_ringbuffer *
 703 msm_ringbuffer_new_object(struct fd_pipe *pipe, uint32_t size)
 704 {
 705         struct msm_ringbuffer *msm_ring = malloc(sizeof(*msm_ring));
 706
 707         msm_ring->u.pipe = pipe;
 708         msm_ring->offset = 0;
 709         msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size, 0);
 710         msm_ring->base.refcnt = 1;
 711
 712         msm_ring->u.reloc_bos = NULL;
 713         msm_ring->u.nr_reloc_bos = msm_ring->u.max_reloc_bos = 0;
 714
 715         msm_ring->u.ring_set = _mesa_set_create(NULL,
 716                         _mesa_hash_pointer, _mesa_key_pointer_equal);
 717
 718         return msm_ringbuffer_init(msm_ring, size, _FD_RINGBUFFER_OBJECT);
 719 }