src/gallium/drivers/iris/iris_fence.c

   1 /*
   2  * Copyright © 2018 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20  * DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 /**
  24  * @file iris_fence.c
  25  *
  26  * Fences for driver and IPC serialisation, scheduling and synchronisation.
  27  */
  28
  29 #include "drm-uapi/sync_file.h"
  30 #include "util/u_debug.h"
  31 #include "util/u_inlines.h"
  32 #include "intel/common/gen_gem.h"
  33
  34 #include "iris_batch.h"
  35 #include "iris_bufmgr.h"
  36 #include "iris_context.h"
  37 #include "iris_fence.h"
  38 #include "iris_screen.h"
  39
  40 static uint32_t
  41 gem_syncobj_create(int fd, uint32_t flags)
  42 {
  43    struct drm_syncobj_create args = {
  44       .flags = flags,
  45    };
  46
  47    gen_ioctl(fd, DRM_IOCTL_SYNCOBJ_CREATE, &args);
  48
  49    return args.handle;
  50 }
  51
  52 static void
  53 gem_syncobj_destroy(int fd, uint32_t handle)
  54 {
  55    struct drm_syncobj_destroy args = {
  56       .handle = handle,
  57    };
  58
  59    gen_ioctl(fd, DRM_IOCTL_SYNCOBJ_DESTROY, &args);
  60 }
  61
  62 /**
  63  * Make a new sync-point.
  64  */
  65 struct iris_syncobj *
  66 iris_create_syncobj(struct iris_screen *screen)
  67 {
  68    struct iris_syncobj *syncobj = malloc(sizeof(*syncobj));
  69
  70    if (!syncobj)
  71       return NULL;
  72
  73    syncobj->handle = gem_syncobj_create(screen->fd, 0);
  74    assert(syncobj->handle);
  75
  76    pipe_reference_init(&syncobj->ref, 1);
  77
  78    return syncobj;
  79 }
  80
  81 void
  82 iris_syncobj_destroy(struct iris_screen *screen, struct iris_syncobj *syncobj)
  83 {
  84    gem_syncobj_destroy(screen->fd, syncobj->handle);
  85    free(syncobj);
  86 }
  87
  88 /**
  89  * Add a sync-point to the batch, with the given flags.
  90  *
  91  * \p flags   One of I915_EXEC_FENCE_WAIT or I915_EXEC_FENCE_SIGNAL.
  92  */
  93 void
  94 iris_batch_add_syncobj(struct iris_batch *batch,
  95                        struct iris_syncobj *syncobj,
  96                        unsigned flags)
  97 {
  98    struct drm_i915_gem_exec_fence *fence =
  99       util_dynarray_grow(&batch->exec_fences, struct drm_i915_gem_exec_fence, 1);
 100
 101    *fence = (struct drm_i915_gem_exec_fence) {
 102       .handle = syncobj->handle,
 103       .flags = flags,
 104    };
 105
 106    struct iris_syncobj **store =
 107       util_dynarray_grow(&batch->syncobjs, struct iris_syncobj *, 1);
 108
 109    *store = NULL;
 110    iris_syncobj_reference(batch->screen, store, syncobj);
 111 }
 112
 113 /**
 114  * Walk through a batch's dependencies (any I915_EXEC_FENCE_WAIT syncobjs)
 115  * and unreference any which have already passed.
 116  *
 117  * Sometimes the compute batch is seldom used, and accumulates references
 118  * to stale render batches that are no longer of interest, so we can free
 119  * those up.
 120  */
 121 static void
 122 clear_stale_syncobjs(struct iris_batch *batch)
 123 {
 124    struct iris_screen *screen = batch->screen;
 125
 126    int n = util_dynarray_num_elements(&batch->syncobjs, struct iris_syncobj *);
 127
 128    assert(n == util_dynarray_num_elements(&batch->exec_fences,
 129                                           struct drm_i915_gem_exec_fence));
 130
 131    /* Skip the first syncobj, as it's the signalling one. */
 132    for (int i = n - 1; i > 1; i--) {
 133       struct iris_syncobj **syncobj =
 134          util_dynarray_element(&batch->syncobjs, struct iris_syncobj *, i);
 135       struct drm_i915_gem_exec_fence *fence =
 136          util_dynarray_element(&batch->exec_fences,
 137                                struct drm_i915_gem_exec_fence, i);
 138       assert(fence->flags & I915_EXEC_FENCE_WAIT);
 139
 140       if (iris_wait_syncobj(&screen->base, *syncobj, 0))
 141          continue;
 142
 143       /* This sync object has already passed, there's no need to continue
 144        * marking it as a dependency; we can stop holding on to the reference.
 145        */
 146       iris_syncobj_reference(screen, syncobj, NULL);
 147
 148       /* Remove it from the lists; move the last element here. */
 149       struct iris_syncobj **nth_syncobj =
 150          util_dynarray_pop_ptr(&batch->syncobjs, struct iris_syncobj *);
 151       struct drm_i915_gem_exec_fence *nth_fence =
 152          util_dynarray_pop_ptr(&batch->exec_fences,
 153                                struct drm_i915_gem_exec_fence);
 154
 155       if (syncobj != nth_syncobj) {
 156          *syncobj = *nth_syncobj;
 157          memcpy(nth_fence, fence, sizeof(*fence));
 158       }
 159    }
 160 }
 161
 162 /* ------------------------------------------------------------------- */
 163
 164 struct pipe_fence_handle {
 165    struct pipe_reference ref;
 166
 167    struct pipe_context *unflushed_ctx;
 168
 169    struct iris_fine_fence *fine[IRIS_BATCH_COUNT];
 170 };
 171
 172 static void
 173 iris_fence_destroy(struct pipe_screen *p_screen,
 174                    struct pipe_fence_handle *fence)
 175 {
 176    struct iris_screen *screen = (struct iris_screen *)p_screen;
 177
 178    for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++)
 179       iris_fine_fence_reference(screen, &fence->fine[i], NULL);
 180
 181    free(fence);
 182 }
 183
 184 static void
 185 iris_fence_reference(struct pipe_screen *p_screen,
 186                      struct pipe_fence_handle **dst,
 187                      struct pipe_fence_handle *src)
 188 {
 189    if (pipe_reference(*dst ? &(*dst)->ref : NULL,
 190                       src ? &src->ref : NULL))
 191       iris_fence_destroy(p_screen, *dst);
 192
 193    *dst = src;
 194 }
 195
 196 bool
 197 iris_wait_syncobj(struct pipe_screen *p_screen,
 198                   struct iris_syncobj *syncobj,
 199                   int64_t timeout_nsec)
 200 {
 201    if (!syncobj)
 202       return false;
 203
 204    struct iris_screen *screen = (struct iris_screen *)p_screen;
 205    struct drm_syncobj_wait args = {
 206       .handles = (uintptr_t)&syncobj->handle,
 207       .count_handles = 1,
 208       .timeout_nsec = timeout_nsec,
 209    };
 210    return gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args);
 211 }
 212
 213 #define CSI "\e["
 214 #define BLUE_HEADER  CSI "0;97;44m"
 215 #define NORMAL       CSI "0m"
 216
 217 static void
 218 iris_fence_flush(struct pipe_context *ctx,
 219                  struct pipe_fence_handle **out_fence,
 220                  unsigned flags)
 221 {
 222    struct iris_screen *screen = (void *) ctx->screen;
 223    struct iris_context *ice = (struct iris_context *)ctx;
 224
 225    /* We require DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT (kernel 5.2+) for
 226     * deferred flushes.  Just ignore the request to defer on older kernels.
 227     */
 228    if (!(screen->kernel_features & KERNEL_HAS_WAIT_FOR_SUBMIT))
 229       flags &= ~PIPE_FLUSH_DEFERRED;
 230
 231    const bool deferred = flags & PIPE_FLUSH_DEFERRED;
 232
 233    if (flags & PIPE_FLUSH_END_OF_FRAME) {
 234       ice->frame++;
 235
 236       if (INTEL_DEBUG & DEBUG_SUBMIT) {
 237          fprintf(stderr, "%s ::: FRAME %-10u (ctx %p)%-35c%s\n",
 238                  (INTEL_DEBUG & DEBUG_COLOR) ? BLUE_HEADER : "",
 239                  ice->frame, ctx, ' ',
 240                  (INTEL_DEBUG & DEBUG_COLOR) ? NORMAL : "");
 241       }
 242    }
 243
 244    if (!deferred) {
 245       for (unsigned i = 0; i < IRIS_BATCH_COUNT; i++)
 246          iris_batch_flush(&ice->batches[i]);
 247    }
 248
 249    if (!out_fence)
 250       return;
 251
 252    struct pipe_fence_handle *fence = calloc(1, sizeof(*fence));
 253    if (!fence)
 254       return;
 255
 256    pipe_reference_init(&fence->ref, 1);
 257
 258    if (deferred)
 259       fence->unflushed_ctx = ctx;
 260
 261    for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) {
 262       struct iris_batch *batch = &ice->batches[b];
 263
 264       if (deferred && iris_batch_bytes_used(batch) > 0) {
 265          struct iris_fine_fence *fine =
 266             iris_fine_fence_new(batch, IRIS_FENCE_BOTTOM_OF_PIPE);
 267          iris_fine_fence_reference(screen, &fence->fine[b], fine);
 268          iris_fine_fence_reference(screen, &fine, NULL);
 269       } else {
 270          /* This batch has no commands queued up (perhaps we just flushed,
 271           * or all the commands are on the other batch).  Wait for the last
 272           * syncobj on this engine - unless it's already finished by now.
 273           */
 274          if (iris_fine_fence_signaled(batch->last_fence))
 275             continue;
 276
 277          iris_fine_fence_reference(screen, &fence->fine[b], batch->last_fence);
 278       }
 279    }
 280
 281    iris_fence_reference(ctx->screen, out_fence, NULL);
 282    *out_fence = fence;
 283 }
 284
 285 static void
 286 iris_fence_await(struct pipe_context *ctx,
 287                  struct pipe_fence_handle *fence)
 288 {
 289    struct iris_context *ice = (struct iris_context *)ctx;
 290
 291    /* Unflushed fences from the same context are no-ops. */
 292    if (ctx && ctx == fence->unflushed_ctx)
 293       return;
 294
 295    /* XXX: We can't safely flush the other context, because it might be
 296     *      bound to another thread, and poking at its internals wouldn't
 297     *      be safe.  In the future we should use MI_SEMAPHORE_WAIT and
 298     *      block until the other job has been submitted, relying on
 299     *      kernel timeslicing to preempt us until the other job is
 300     *      actually flushed and the seqno finally passes.
 301     */
 302    if (fence->unflushed_ctx) {
 303       pipe_debug_message(&ice->dbg, CONFORMANCE, "%s",
 304                          "glWaitSync on unflushed fence from another context "
 305                          "is unlikely to work without kernel 5.8+\n");
 306    }
 307
 308    for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) {
 309       struct iris_fine_fence *fine = fence->fine[i];
 310
 311       if (iris_fine_fence_signaled(fine))
 312          continue;
 313
 314       for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) {
 315          struct iris_batch *batch = &ice->batches[b];
 316
 317          /* We're going to make any future work in this batch wait for our
 318           * fence to have gone by.  But any currently queued work doesn't
 319           * need to wait.  Flush the batch now, so it can happen sooner.
 320           */
 321          iris_batch_flush(batch);
 322
 323          /* Before adding a new reference, clean out any stale ones. */
 324          clear_stale_syncobjs(batch);
 325
 326          iris_batch_add_syncobj(batch, fine->syncobj, I915_EXEC_FENCE_WAIT);
 327       }
 328    }
 329 }
 330
 331 #define NSEC_PER_SEC (1000 * USEC_PER_SEC)
 332 #define USEC_PER_SEC (1000 * MSEC_PER_SEC)
 333 #define MSEC_PER_SEC (1000)
 334
 335 static uint64_t
 336 gettime_ns(void)
 337 {
 338    struct timespec current;
 339    clock_gettime(CLOCK_MONOTONIC, &current);
 340    return (uint64_t)current.tv_sec * NSEC_PER_SEC + current.tv_nsec;
 341 }
 342
 343 static uint64_t
 344 rel2abs(uint64_t timeout)
 345 {
 346    if (timeout == 0)
 347       return 0;
 348
 349    uint64_t current_time = gettime_ns();
 350    uint64_t max_timeout = (uint64_t) INT64_MAX - current_time;
 351
 352    timeout = MIN2(max_timeout, timeout);
 353
 354    return current_time + timeout;
 355 }
 356
 357 static bool
 358 iris_fence_finish(struct pipe_screen *p_screen,
 359                   struct pipe_context *ctx,
 360                   struct pipe_fence_handle *fence,
 361                   uint64_t timeout)
 362 {
 363    struct iris_context *ice = (struct iris_context *)ctx;
 364    struct iris_screen *screen = (struct iris_screen *)p_screen;
 365
 366    /* If we created the fence with PIPE_FLUSH_DEFERRED, we may not have
 367     * flushed yet.  Check if our syncobj is the current batch's signalling
 368     * syncobj - if so, we haven't flushed and need to now.
 369     *
 370     * The Gallium docs mention that a flush will occur if \p ctx matches
 371     * the context the fence was created with.  It may be NULL, so we check
 372     * that it matches first.
 373     */
 374    if (ctx && ctx == fence->unflushed_ctx) {
 375       for (unsigned i = 0; i < IRIS_BATCH_COUNT; i++) {
 376          struct iris_fine_fence *fine = fence->fine[i];
 377
 378          if (iris_fine_fence_signaled(fine))
 379             continue;
 380
 381          if (fine->syncobj == iris_batch_get_signal_syncobj(&ice->batches[i]))
 382             iris_batch_flush(&ice->batches[i]);
 383       }
 384
 385       /* The fence is no longer deferred. */
 386       fence->unflushed_ctx = NULL;
 387    }
 388
 389    unsigned int handle_count = 0;
 390    uint32_t handles[ARRAY_SIZE(fence->fine)];
 391    for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) {
 392       struct iris_fine_fence *fine = fence->fine[i];
 393
 394       if (iris_fine_fence_signaled(fine))
 395          continue;
 396
 397       handles[handle_count++] = fine->syncobj->handle;
 398    }
 399
 400    if (handle_count == 0)
 401       return true;
 402
 403    struct drm_syncobj_wait args = {
 404       .handles = (uintptr_t)handles,
 405       .count_handles = handle_count,
 406       .timeout_nsec = rel2abs(timeout),
 407       .flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL
 408    };
 409
 410    if (fence->unflushed_ctx) {
 411       /* This fence had a deferred flush from another context.  We can't
 412        * safely flush it here, because the context might be bound to a
 413        * different thread, and poking at its internals wouldn't be safe.
 414        *
 415        * Instead, use the WAIT_FOR_SUBMIT flag to block and hope that
 416        * another thread submits the work.
 417        */
 418       args.flags |= DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT;
 419    }
 420
 421    return gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args) == 0;
 422 }
 423
 424 static int
 425 sync_merge_fd(int sync_fd, int new_fd)
 426 {
 427    if (sync_fd == -1)
 428       return new_fd;
 429
 430    if (new_fd == -1)
 431       return sync_fd;
 432
 433    struct sync_merge_data args = {
 434       .name = "iris fence",
 435       .fd2 = new_fd,
 436       .fence = -1,
 437    };
 438
 439    gen_ioctl(sync_fd, SYNC_IOC_MERGE, &args);
 440    close(new_fd);
 441    close(sync_fd);
 442
 443    return args.fence;
 444 }
 445
 446 static int
 447 iris_fence_get_fd(struct pipe_screen *p_screen,
 448                   struct pipe_fence_handle *fence)
 449 {
 450    struct iris_screen *screen = (struct iris_screen *)p_screen;
 451    int fd = -1;
 452
 453    /* Deferred fences aren't supported. */
 454    if (fence->unflushed_ctx)
 455       return -1;
 456
 457    for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) {
 458       struct iris_fine_fence *fine = fence->fine[i];
 459
 460       if (iris_fine_fence_signaled(fine))
 461          continue;
 462
 463       struct drm_syncobj_handle args = {
 464          .handle = fine->syncobj->handle,
 465          .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE,
 466          .fd = -1,
 467       };
 468
 469       gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
 470       fd = sync_merge_fd(fd, args.fd);
 471    }
 472
 473    if (fd == -1) {
 474       /* Our fence has no syncobj's recorded.  This means that all of the
 475        * batches had already completed, their syncobj's had been signalled,
 476        * and so we didn't bother to record them.  But we're being asked to
 477        * export such a fence.  So export a dummy already-signalled syncobj.
 478        */
 479       struct drm_syncobj_handle args = {
 480          .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE, .fd = -1,
 481       };
 482
 483       args.handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED);
 484       gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
 485       gem_syncobj_destroy(screen->fd, args.handle);
 486       return args.fd;
 487    }
 488
 489    return fd;
 490 }
 491
 492 static void
 493 iris_fence_create_fd(struct pipe_context *ctx,
 494                      struct pipe_fence_handle **out,
 495                      int fd,
 496                      enum pipe_fd_type type)
 497 {
 498    assert(type == PIPE_FD_TYPE_NATIVE_SYNC);
 499
 500    struct iris_screen *screen = (struct iris_screen *)ctx->screen;
 501    struct drm_syncobj_handle args = {
 502       .handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED),
 503       .flags = DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE,
 504       .fd = fd,
 505    };
 506    if (gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, &args) == -1) {
 507       fprintf(stderr, "DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE failed: %s\n",
 508               strerror(errno));
 509       gem_syncobj_destroy(screen->fd, args.handle);
 510       *out = NULL;
 511       return;
 512    }
 513
 514    struct iris_syncobj *syncobj = malloc(sizeof(*syncobj));
 515    if (!syncobj) {
 516       *out = NULL;
 517       return;
 518    }
 519    syncobj->handle = args.handle;
 520    pipe_reference_init(&syncobj->ref, 1);
 521
 522    struct iris_fine_fence *fine = calloc(1, sizeof(*fine));
 523    if (!fine) {
 524       free(syncobj);
 525       *out = NULL;
 526       return;
 527    }
 528
 529    static const uint32_t zero = 0;
 530
 531    /* Fences work in terms of iris_fine_fence, but we don't actually have a
 532     * seqno for an imported fence.  So, create a fake one which always
 533     * returns as 'not signaled' so we fall back to using the sync object.
 534     */
 535    fine->seqno = UINT32_MAX;
 536    fine->map = &zero;
 537    fine->syncobj = syncobj;
 538    fine->flags = IRIS_FENCE_END;
 539    pipe_reference_init(&fine->reference, 1);
 540
 541    struct pipe_fence_handle *fence = calloc(1, sizeof(*fence));
 542    if (!fence) {
 543       free(fine);
 544       free(syncobj);
 545       *out = NULL;
 546       return;
 547    }
 548    pipe_reference_init(&fence->ref, 1);
 549    fence->fine[0] = fine;
 550
 551    *out = fence;
 552 }
 553
 554 void
 555 iris_init_screen_fence_functions(struct pipe_screen *screen)
 556 {
 557    screen->fence_reference = iris_fence_reference;
 558    screen->fence_finish = iris_fence_finish;
 559    screen->fence_get_fd = iris_fence_get_fd;
 560 }
 561
 562 void
 563 iris_init_context_fence_functions(struct pipe_context *ctx)
 564 {
 565    ctx->flush = iris_fence_flush;
 566    ctx->create_fence_fd = iris_fence_create_fd;
 567    ctx->fence_server_sync = iris_fence_await;
 568 }