src/freedreno/drm/msm_ringbuffer.c

   1 /*
   2  * Copyright (C) 2012-2018 Rob Clark <robclark@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark <robclark@freedesktop.org>
  25  */
  26
  27 #include <assert.h>
  28 #include <inttypes.h>
  29
  30 #include "util/hash_table.h"
  31 #include "util/set.h"
  32 #include "util/slab.h"
  33
  34 #include "drm/freedreno_ringbuffer.h"
  35 #include "msm_priv.h"
  36
  37 /* The legacy implementation of submit/ringbuffer, which still does the
  38  * traditional reloc and cmd tracking
  39  */
  40
  41
  42 #define INIT_SIZE 0x1000
  43
  44
  45 struct msm_submit {
  46         struct fd_submit base;
  47
  48         DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos);
  49         DECLARE_ARRAY(struct fd_bo *, bos);
  50
  51         /* maps fd_bo to idx in bos table: */
  52         struct hash_table *bo_table;
  53
  54         struct slab_mempool ring_pool;
  55
  56         /* hash-set of associated rings: */
  57         struct set *ring_set;
  58
  59         struct fd_ringbuffer *primary;
  60
  61         /* Allow for sub-allocation of stateobj ring buffers (ie. sharing
  62          * the same underlying bo)..
  63          *
  64          * We also rely on previous stateobj having been fully constructed
  65          * so we can reclaim extra space at it's end.
  66          */
  67         struct fd_ringbuffer *suballoc_ring;
  68 };
  69 FD_DEFINE_CAST(fd_submit, msm_submit);
  70
  71 /* for FD_RINGBUFFER_GROWABLE rb's, tracks the 'finalized' cmdstream buffers
  72  * and sizes.  Ie. a finalized buffer can have no more commands appended to
  73  * it.
  74  */
  75 struct msm_cmd {
  76         struct fd_bo *ring_bo;
  77         unsigned size;
  78         DECLARE_ARRAY(struct drm_msm_gem_submit_reloc, relocs);
  79 };
  80
  81 static struct msm_cmd *
  82 cmd_new(struct fd_bo *ring_bo)
  83 {
  84         struct msm_cmd *cmd = malloc(sizeof(*cmd));
  85         cmd->ring_bo = fd_bo_ref(ring_bo);
  86         cmd->size = 0;
  87         cmd->nr_relocs = cmd->max_relocs = 0;
  88         cmd->relocs = NULL;
  89         return cmd;
  90 }
  91
  92 static void
  93 cmd_free(struct msm_cmd *cmd)
  94 {
  95         fd_bo_del(cmd->ring_bo);
  96         free(cmd->relocs);
  97         free(cmd);
  98 }
  99
 100 /* for _FD_RINGBUFFER_OBJECT rb's we need to track the bo's and flags to
 101  * later copy into the submit when the stateobj rb is later referenced by
 102  * a regular rb:
 103  */
 104 struct msm_reloc_bo {
 105         struct fd_bo *bo;
 106         unsigned flags;
 107 };
 108
 109 struct msm_ringbuffer {
 110         struct fd_ringbuffer base;
 111
 112         /* for FD_RINGBUFFER_STREAMING rb's which are sub-allocated */
 113         unsigned offset;
 114
 115         union {
 116                 /* for _FD_RINGBUFFER_OBJECT case: */
 117                 struct {
 118                         struct fd_pipe *pipe;
 119                         DECLARE_ARRAY(struct msm_reloc_bo, reloc_bos);
 120                         struct set *ring_set;
 121                 };
 122                 /* for other cases: */
 123                 struct {
 124                         struct fd_submit *submit;
 125                         DECLARE_ARRAY(struct msm_cmd *, cmds);
 126                 };
 127         } u;
 128
 129         struct msm_cmd *cmd;          /* current cmd */
 130         struct fd_bo *ring_bo;
 131 };
 132 FD_DEFINE_CAST(fd_ringbuffer, msm_ringbuffer);
 133
 134 static void finalize_current_cmd(struct fd_ringbuffer *ring);
 135 static struct fd_ringbuffer * msm_ringbuffer_init(
 136                 struct msm_ringbuffer *msm_ring,
 137                 uint32_t size, enum fd_ringbuffer_flags flags);
 138
 139 /* add (if needed) bo to submit and return index: */
 140 static uint32_t
 141 append_bo(struct msm_submit *submit, struct fd_bo *bo, uint32_t flags)
 142 {
 143         struct msm_bo *msm_bo = to_msm_bo(bo);
 144         uint32_t idx;
 145
 146         /* NOTE: it is legal to use the same bo on different threads for
 147          * different submits.  But it is not legal to use the same submit
 148          * from given threads.
 149          */
 150         idx = READ_ONCE(msm_bo->idx);
 151
 152         if (unlikely((idx >= submit->nr_submit_bos) ||
 153                         (submit->submit_bos[idx].handle != bo->handle))) {
 154                 uint32_t hash = _mesa_hash_pointer(bo);
 155                 struct hash_entry *entry;
 156
 157                 entry = _mesa_hash_table_search_pre_hashed(submit->bo_table, hash, bo);
 158                 if (entry) {
 159                         /* found */
 160                         idx = (uint32_t)(uintptr_t)entry->data;
 161                 } else {
 162                         idx = APPEND(submit, submit_bos);
 163                         idx = APPEND(submit, bos);
 164
 165                         submit->submit_bos[idx].flags = bo->flags;
 166                         submit->submit_bos[idx].handle = bo->handle;
 167                         submit->submit_bos[idx].presumed = 0;
 168
 169                         submit->bos[idx] = fd_bo_ref(bo);
 170
 171                         _mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo,
 172                                         (void *)(uintptr_t)idx);
 173                 }
 174                 msm_bo->idx = idx;
 175         }
 176
 177         if (flags & FD_RELOC_WRITE)
 178                 submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
 179
 180         return idx;
 181 }
 182
 183 static void
 184 append_ring(struct set *set, struct fd_ringbuffer *ring)
 185 {
 186         uint32_t hash = _mesa_hash_pointer(ring);
 187
 188         if (!_mesa_set_search_pre_hashed(set, hash, ring)) {
 189                 fd_ringbuffer_ref(ring);
 190                 _mesa_set_add_pre_hashed(set, hash, ring);
 191         }
 192 }
 193
 194 static void
 195 msm_submit_suballoc_ring_bo(struct fd_submit *submit,
 196                 struct msm_ringbuffer *msm_ring, uint32_t size)
 197 {
 198         struct msm_submit *msm_submit = to_msm_submit(submit);
 199         unsigned suballoc_offset = 0;
 200         struct fd_bo *suballoc_bo = NULL;
 201
 202         if (msm_submit->suballoc_ring) {
 203                 struct msm_ringbuffer *suballoc_ring =
 204                                 to_msm_ringbuffer(msm_submit->suballoc_ring);
 205
 206                 suballoc_bo = suballoc_ring->ring_bo;
 207                 suballoc_offset = fd_ringbuffer_size(msm_submit->suballoc_ring) +
 208                                 suballoc_ring->offset;
 209
 210                 suballoc_offset = align(suballoc_offset, 0x10);
 211
 212                 if ((size + suballoc_offset) > suballoc_bo->size) {
 213                         suballoc_bo = NULL;
 214                 }
 215         }
 216
 217         if (!suballoc_bo) {
 218                 // TODO possibly larger size for streaming bo?
 219                 msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, 0x8000);
 220                 msm_ring->offset = 0;
 221         } else {
 222                 msm_ring->ring_bo = fd_bo_ref(suballoc_bo);
 223                 msm_ring->offset = suballoc_offset;
 224         }
 225
 226         struct fd_ringbuffer *old_suballoc_ring = msm_submit->suballoc_ring;
 227
 228         msm_submit->suballoc_ring = fd_ringbuffer_ref(&msm_ring->base);
 229
 230         if (old_suballoc_ring)
 231                 fd_ringbuffer_del(old_suballoc_ring);
 232 }
 233
 234 static struct fd_ringbuffer *
 235 msm_submit_new_ringbuffer(struct fd_submit *submit, uint32_t size,
 236                 enum fd_ringbuffer_flags flags)
 237 {
 238         struct msm_submit *msm_submit = to_msm_submit(submit);
 239         struct msm_ringbuffer *msm_ring;
 240
 241         msm_ring = slab_alloc_st(&msm_submit->ring_pool);
 242
 243         msm_ring->u.submit = submit;
 244
 245         /* NOTE: needs to be before _suballoc_ring_bo() since it could
 246          * increment the refcnt of the current ring
 247          */
 248         msm_ring->base.refcnt = 1;
 249
 250         if (flags & FD_RINGBUFFER_STREAMING) {
 251                 msm_submit_suballoc_ring_bo(submit, msm_ring, size);
 252         } else {
 253                 if (flags & FD_RINGBUFFER_GROWABLE)
 254                         size = INIT_SIZE;
 255
 256                 msm_ring->offset = 0;
 257                 msm_ring->ring_bo = fd_bo_new_ring(submit->pipe->dev, size);
 258         }
 259
 260         if (!msm_ringbuffer_init(msm_ring, size, flags))
 261                 return NULL;
 262
 263         if (flags & FD_RINGBUFFER_PRIMARY) {
 264                 debug_assert(!msm_submit->primary);
 265                 msm_submit->primary = fd_ringbuffer_ref(&msm_ring->base);
 266         }
 267
 268         return &msm_ring->base;
 269 }
 270
 271 static struct drm_msm_gem_submit_reloc *
 272 handle_stateobj_relocs(struct msm_submit *submit, struct msm_ringbuffer *ring)
 273 {
 274         struct msm_cmd *cmd = ring->cmd;
 275         struct drm_msm_gem_submit_reloc *relocs;
 276
 277         relocs = malloc(cmd->nr_relocs * sizeof(*relocs));
 278
 279         for (unsigned i = 0; i < cmd->nr_relocs; i++) {
 280                 unsigned idx = cmd->relocs[i].reloc_idx;
 281                 struct fd_bo *bo = ring->u.reloc_bos[idx].bo;
 282                 unsigned flags = 0;
 283
 284                 if (ring->u.reloc_bos[idx].flags & MSM_SUBMIT_BO_WRITE)
 285                         flags |= FD_RELOC_WRITE;
 286
 287                 relocs[i] = cmd->relocs[i];
 288                 relocs[i].reloc_idx = append_bo(submit, bo, flags);
 289         }
 290
 291         return relocs;
 292 }
 293
 294 static int
 295 msm_submit_flush(struct fd_submit *submit, int in_fence_fd,
 296                 int *out_fence_fd, uint32_t *out_fence)
 297 {
 298         struct msm_submit *msm_submit = to_msm_submit(submit);
 299         struct msm_pipe *msm_pipe = to_msm_pipe(submit->pipe);
 300         struct drm_msm_gem_submit req = {
 301                         .flags = msm_pipe->pipe,
 302                         .queueid = msm_pipe->queue_id,
 303         };
 304         int ret;
 305
 306         debug_assert(msm_submit->primary);
 307
 308         finalize_current_cmd(msm_submit->primary);
 309         append_ring(msm_submit->ring_set, msm_submit->primary);
 310
 311         unsigned nr_cmds = 0;
 312         unsigned nr_objs = 0;
 313
 314         set_foreach(msm_submit->ring_set, entry) {
 315                 struct fd_ringbuffer *ring = (void *)entry->key;
 316                 if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 317                         nr_cmds += 1;
 318                         nr_objs += 1;
 319                 } else {
 320                         if (ring != msm_submit->primary)
 321                                 finalize_current_cmd(ring);
 322                         nr_cmds += to_msm_ringbuffer(ring)->u.nr_cmds;
 323                 }
 324         }
 325
 326         void *obj_relocs[nr_objs];
 327         struct drm_msm_gem_submit_cmd cmds[nr_cmds];
 328         unsigned i = 0, o = 0;
 329
 330         set_foreach(msm_submit->ring_set, entry) {
 331                 struct fd_ringbuffer *ring = (void *)entry->key;
 332                 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 333
 334                 debug_assert(i < nr_cmds);
 335
 336                 // TODO handle relocs:
 337                 if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 338
 339                         debug_assert(o < nr_objs);
 340
 341                         void *relocs = handle_stateobj_relocs(msm_submit, msm_ring);
 342                         obj_relocs[o++] = relocs;
 343
 344                         cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
 345                         cmds[i].submit_idx =
 346                                 append_bo(msm_submit, msm_ring->ring_bo, 0);
 347                         cmds[i].submit_offset = msm_ring->offset;
 348                         cmds[i].size = offset_bytes(ring->cur, ring->start);
 349                         cmds[i].pad = 0;
 350                         cmds[i].nr_relocs = msm_ring->cmd->nr_relocs;
 351                         cmds[i].relocs = VOID2U64(relocs);
 352
 353                         i++;
 354                 } else {
 355                         for (unsigned j = 0; j < msm_ring->u.nr_cmds; j++) {
 356                                 if (ring->flags & FD_RINGBUFFER_PRIMARY) {
 357                                         cmds[i].type = MSM_SUBMIT_CMD_BUF;
 358                                 } else {
 359                                         cmds[i].type = MSM_SUBMIT_CMD_IB_TARGET_BUF;
 360                                 }
 361                                 cmds[i].submit_idx = append_bo(msm_submit,
 362                                                 msm_ring->u.cmds[j]->ring_bo, 0);
 363                                 cmds[i].submit_offset = msm_ring->offset;
 364                                 cmds[i].size = msm_ring->u.cmds[j]->size;
 365                                 cmds[i].pad = 0;
 366                                 cmds[i].nr_relocs = msm_ring->u.cmds[j]->nr_relocs;
 367                                 cmds[i].relocs = VOID2U64(msm_ring->u.cmds[j]->relocs);
 368
 369                                 i++;
 370                         }
 371                 }
 372         }
 373
 374         if (in_fence_fd != -1) {
 375                 req.flags |= MSM_SUBMIT_FENCE_FD_IN | MSM_SUBMIT_NO_IMPLICIT;
 376                 req.fence_fd = in_fence_fd;
 377         }
 378
 379         if (out_fence_fd) {
 380                 req.flags |= MSM_SUBMIT_FENCE_FD_OUT;
 381         }
 382
 383         /* needs to be after get_cmd() as that could create bos/cmds table: */
 384         req.bos = VOID2U64(msm_submit->submit_bos),
 385         req.nr_bos = msm_submit->nr_submit_bos;
 386         req.cmds = VOID2U64(cmds),
 387         req.nr_cmds = nr_cmds;
 388
 389         DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos);
 390
 391         ret = drmCommandWriteRead(submit->pipe->dev->fd, DRM_MSM_GEM_SUBMIT,
 392                         &req, sizeof(req));
 393         if (ret) {
 394                 ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno));
 395                 msm_dump_submit(&req);
 396         } else if (!ret) {
 397                 if (out_fence)
 398                         *out_fence = req.fence;
 399
 400                 if (out_fence_fd)
 401                         *out_fence_fd = req.fence_fd;
 402         }
 403
 404         for (unsigned o = 0; o < nr_objs; o++)
 405                 free(obj_relocs[o]);
 406
 407         return ret;
 408 }
 409
 410 static void
 411 unref_rings(struct set_entry *entry)
 412 {
 413         struct fd_ringbuffer *ring = (void *)entry->key;
 414         fd_ringbuffer_del(ring);
 415 }
 416
 417 static void
 418 msm_submit_destroy(struct fd_submit *submit)
 419 {
 420         struct msm_submit *msm_submit = to_msm_submit(submit);
 421
 422         if (msm_submit->primary)
 423                 fd_ringbuffer_del(msm_submit->primary);
 424         if (msm_submit->suballoc_ring)
 425                 fd_ringbuffer_del(msm_submit->suballoc_ring);
 426
 427         _mesa_hash_table_destroy(msm_submit->bo_table, NULL);
 428         _mesa_set_destroy(msm_submit->ring_set, unref_rings);
 429
 430         // TODO it would be nice to have a way to debug_assert() if all
 431         // rb's haven't been free'd back to the slab, because that is
 432         // an indication that we are leaking bo's
 433         slab_destroy(&msm_submit->ring_pool);
 434
 435         for (unsigned i = 0; i < msm_submit->nr_bos; i++)
 436                 fd_bo_del(msm_submit->bos[i]);
 437
 438         free(msm_submit->submit_bos);
 439         free(msm_submit->bos);
 440         free(msm_submit);
 441 }
 442
 443 static const struct fd_submit_funcs submit_funcs = {
 444                 .new_ringbuffer = msm_submit_new_ringbuffer,
 445                 .flush = msm_submit_flush,
 446                 .destroy = msm_submit_destroy,
 447 };
 448
 449 struct fd_submit *
 450 msm_submit_new(struct fd_pipe *pipe)
 451 {
 452         struct msm_submit *msm_submit = calloc(1, sizeof(*msm_submit));
 453         struct fd_submit *submit;
 454
 455         msm_submit->bo_table = _mesa_hash_table_create(NULL,
 456                         _mesa_hash_pointer, _mesa_key_pointer_equal);
 457         msm_submit->ring_set = _mesa_set_create(NULL,
 458                         _mesa_hash_pointer, _mesa_key_pointer_equal);
 459         // TODO tune size:
 460         slab_create(&msm_submit->ring_pool, sizeof(struct msm_ringbuffer), 16);
 461
 462         submit = &msm_submit->base;
 463         submit->pipe = pipe;
 464         submit->funcs = &submit_funcs;
 465
 466         return submit;
 467 }
 468
 469
 470 static void
 471 finalize_current_cmd(struct fd_ringbuffer *ring)
 472 {
 473         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 474
 475         debug_assert(!(ring->flags & _FD_RINGBUFFER_OBJECT));
 476
 477         if (!msm_ring->cmd)
 478                 return;
 479
 480         debug_assert(msm_ring->cmd->ring_bo == msm_ring->ring_bo);
 481
 482         unsigned idx = APPEND(&msm_ring->u, cmds);
 483
 484         msm_ring->u.cmds[idx] = msm_ring->cmd;
 485         msm_ring->cmd = NULL;
 486
 487         msm_ring->u.cmds[idx]->size = offset_bytes(ring->cur, ring->start);
 488 }
 489
 490 static void
 491 msm_ringbuffer_grow(struct fd_ringbuffer *ring, uint32_t size)
 492 {
 493         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 494         struct fd_pipe *pipe = msm_ring->u.submit->pipe;
 495
 496         debug_assert(ring->flags & FD_RINGBUFFER_GROWABLE);
 497
 498         finalize_current_cmd(ring);
 499
 500         fd_bo_del(msm_ring->ring_bo);
 501         msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size);
 502         msm_ring->cmd = cmd_new(msm_ring->ring_bo);
 503
 504         ring->start = fd_bo_map(msm_ring->ring_bo);
 505         ring->end = &(ring->start[size/4]);
 506         ring->cur = ring->start;
 507         ring->size = size;
 508 }
 509
 510 static void
 511 msm_ringbuffer_emit_reloc(struct fd_ringbuffer *ring,
 512                 const struct fd_reloc *reloc)
 513 {
 514         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 515         struct fd_pipe *pipe;
 516         unsigned reloc_idx;
 517
 518         if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 519                 unsigned idx = APPEND(&msm_ring->u, reloc_bos);
 520
 521                 msm_ring->u.reloc_bos[idx].bo = fd_bo_ref(reloc->bo);
 522                 msm_ring->u.reloc_bos[idx].flags = reloc->flags;
 523
 524                 /* this gets fixed up at submit->flush() time, since this state-
 525                  * object rb can be used with many different submits
 526                  */
 527                 reloc_idx = idx;
 528
 529                 pipe = msm_ring->u.pipe;
 530         } else {
 531                 struct msm_submit *msm_submit =
 532                                 to_msm_submit(msm_ring->u.submit);
 533
 534                 reloc_idx = append_bo(msm_submit, reloc->bo, reloc->flags);
 535
 536                 pipe = msm_ring->u.submit->pipe;
 537         }
 538
 539         struct drm_msm_gem_submit_reloc *r;
 540         unsigned idx = APPEND(msm_ring->cmd, relocs);
 541
 542         r = &msm_ring->cmd->relocs[idx];
 543
 544         r->reloc_idx = reloc_idx;
 545         r->reloc_offset = reloc->offset;
 546         r->or = reloc->or;
 547         r->shift = reloc->shift;
 548         r->submit_offset = offset_bytes(ring->cur, ring->start) +
 549                         msm_ring->offset;
 550
 551         ring->cur++;
 552
 553         if (pipe->gpu_id >= 500) {
 554                 idx = APPEND(msm_ring->cmd, relocs);
 555                 r = &msm_ring->cmd->relocs[idx];
 556
 557                 r->reloc_idx = reloc_idx;
 558                 r->reloc_offset = reloc->offset;
 559                 r->or = reloc->orhi;
 560                 r->shift = reloc->shift - 32;
 561                 r->submit_offset = offset_bytes(ring->cur, ring->start) +
 562                                 msm_ring->offset;
 563
 564                 ring->cur++;
 565         }
 566 }
 567
 568 static void
 569 append_stateobj_rings(struct msm_submit *submit, struct fd_ringbuffer *target)
 570 {
 571         struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target);
 572
 573         debug_assert(target->flags & _FD_RINGBUFFER_OBJECT);
 574
 575         set_foreach(msm_target->u.ring_set, entry) {
 576                 struct fd_ringbuffer *ring = (void *)entry->key;
 577
 578                 append_ring(submit->ring_set, ring);
 579
 580                 if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 581                         append_stateobj_rings(submit, ring);
 582                 }
 583         }
 584 }
 585
 586 static uint32_t
 587 msm_ringbuffer_emit_reloc_ring(struct fd_ringbuffer *ring,
 588                 struct fd_ringbuffer *target, uint32_t cmd_idx)
 589 {
 590         struct msm_ringbuffer *msm_target = to_msm_ringbuffer(target);
 591         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 592         struct fd_bo *bo;
 593         uint32_t size;
 594
 595         if ((target->flags & FD_RINGBUFFER_GROWABLE) &&
 596                         (cmd_idx < msm_target->u.nr_cmds)) {
 597                 bo   = msm_target->u.cmds[cmd_idx]->ring_bo;
 598                 size = msm_target->u.cmds[cmd_idx]->size;
 599         } else {
 600                 bo   = msm_target->ring_bo;
 601                 size = offset_bytes(target->cur, target->start);
 602         }
 603
 604         msm_ringbuffer_emit_reloc(ring, &(struct fd_reloc){
 605                 .bo     = bo,
 606                 .flags  = 0,
 607                 .offset = msm_target->offset,
 608         });
 609
 610         if (!size)
 611                 return 0;
 612
 613         if ((target->flags & _FD_RINGBUFFER_OBJECT) &&
 614                         !(ring->flags & _FD_RINGBUFFER_OBJECT)) {
 615                 struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
 616
 617                 append_stateobj_rings(msm_submit, target);
 618         }
 619
 620         if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 621                 append_ring(msm_ring->u.ring_set, target);
 622         } else {
 623                 struct msm_submit *msm_submit = to_msm_submit(msm_ring->u.submit);
 624                 append_ring(msm_submit->ring_set, target);
 625         }
 626
 627         return size;
 628 }
 629
 630 static uint32_t
 631 msm_ringbuffer_cmd_count(struct fd_ringbuffer *ring)
 632 {
 633         if (ring->flags & FD_RINGBUFFER_GROWABLE)
 634                 return to_msm_ringbuffer(ring)->u.nr_cmds + 1;
 635         return 1;
 636 }
 637
 638 static void
 639 msm_ringbuffer_destroy(struct fd_ringbuffer *ring)
 640 {
 641         struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring);
 642
 643         fd_bo_del(msm_ring->ring_bo);
 644         if (msm_ring->cmd)
 645                 cmd_free(msm_ring->cmd);
 646
 647         if (ring->flags & _FD_RINGBUFFER_OBJECT) {
 648                 for (unsigned i = 0; i < msm_ring->u.nr_reloc_bos; i++) {
 649                         fd_bo_del(msm_ring->u.reloc_bos[i].bo);
 650                 }
 651
 652                 _mesa_set_destroy(msm_ring->u.ring_set, unref_rings);
 653
 654                 free(msm_ring->u.reloc_bos);
 655                 free(msm_ring);
 656         } else {
 657                 struct fd_submit *submit = msm_ring->u.submit;
 658
 659                 for (unsigned i = 0; i < msm_ring->u.nr_cmds; i++) {
 660                         cmd_free(msm_ring->u.cmds[i]);
 661                 }
 662
 663                 free(msm_ring->u.cmds);
 664                 slab_free_st(&to_msm_submit(submit)->ring_pool, msm_ring);
 665         }
 666 }
 667
 668 static const struct fd_ringbuffer_funcs ring_funcs = {
 669                 .grow = msm_ringbuffer_grow,
 670                 .emit_reloc = msm_ringbuffer_emit_reloc,
 671                 .emit_reloc_ring = msm_ringbuffer_emit_reloc_ring,
 672                 .cmd_count = msm_ringbuffer_cmd_count,
 673                 .destroy = msm_ringbuffer_destroy,
 674 };
 675
 676 static inline struct fd_ringbuffer *
 677 msm_ringbuffer_init(struct msm_ringbuffer *msm_ring, uint32_t size,
 678                 enum fd_ringbuffer_flags flags)
 679 {
 680         struct fd_ringbuffer *ring = &msm_ring->base;
 681
 682         debug_assert(msm_ring->ring_bo);
 683
 684         uint8_t *base = fd_bo_map(msm_ring->ring_bo);
 685         ring->start = (void *)(base + msm_ring->offset);
 686         ring->end = &(ring->start[size/4]);
 687         ring->cur = ring->start;
 688
 689         ring->size = size;
 690         ring->flags = flags;
 691
 692         ring->funcs = &ring_funcs;
 693
 694         msm_ring->u.cmds = NULL;
 695         msm_ring->u.nr_cmds = msm_ring->u.max_cmds = 0;
 696
 697         msm_ring->cmd = cmd_new(msm_ring->ring_bo);
 698
 699         return ring;
 700 }
 701
 702 struct fd_ringbuffer *
 703 msm_ringbuffer_new_object(struct fd_pipe *pipe, uint32_t size)
 704 {
 705         struct msm_ringbuffer *msm_ring = malloc(sizeof(*msm_ring));
 706
 707         msm_ring->u.pipe = pipe;
 708         msm_ring->offset = 0;
 709         msm_ring->ring_bo = fd_bo_new_ring(pipe->dev, size);
 710         msm_ring->base.refcnt = 1;
 711
 712         msm_ring->u.reloc_bos = NULL;
 713         msm_ring->u.nr_reloc_bos = msm_ring->u.max_reloc_bos = 0;
 714
 715         msm_ring->u.ring_set = _mesa_set_create(NULL,
 716                         _mesa_hash_pointer, _mesa_key_pointer_equal);
 717
 718         return msm_ringbuffer_init(msm_ring, size, _FD_RINGBUFFER_OBJECT);
 719 }