src/gallium/drivers/iris/iris_fence.c

   1 /*
   2  * Copyright © 2018 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20  * DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 /**
  24  * @file iris_fence.c
  25  *
  26  * Fences for driver and IPC serialisation, scheduling and synchronisation.
  27  */
  28
  29 #include <linux/sync_file.h>
  30
  31 #include "util/u_inlines.h"
  32 #include "intel/common/gen_gem.h"
  33
  34 #include "iris_batch.h"
  35 #include "iris_bufmgr.h"
  36 #include "iris_context.h"
  37 #include "iris_fence.h"
  38 #include "iris_screen.h"
  39
  40 static uint32_t
  41 gem_syncobj_create(int fd, uint32_t flags)
  42 {
  43    struct drm_syncobj_create args = {
  44       .flags = flags,
  45    };
  46
  47    gen_ioctl(fd, DRM_IOCTL_SYNCOBJ_CREATE, &args);
  48
  49    return args.handle;
  50 }
  51
  52 static void
  53 gem_syncobj_destroy(int fd, uint32_t handle)
  54 {
  55    struct drm_syncobj_destroy args = {
  56       .handle = handle,
  57    };
  58
  59    gen_ioctl(fd, DRM_IOCTL_SYNCOBJ_DESTROY, &args);
  60 }
  61
  62 /**
  63  * Make a new sync-point.
  64  */
  65 struct iris_syncobj *
  66 iris_create_syncobj(struct iris_screen *screen)
  67 {
  68    struct iris_syncobj *syncobj = malloc(sizeof(*syncobj));
  69
  70    if (!syncobj)
  71       return NULL;
  72
  73    syncobj->handle = gem_syncobj_create(screen->fd, 0);
  74    assert(syncobj->handle);
  75
  76    pipe_reference_init(&syncobj->ref, 1);
  77
  78    return syncobj;
  79 }
  80
  81 void
  82 iris_syncobj_destroy(struct iris_screen *screen, struct iris_syncobj *syncobj)
  83 {
  84    gem_syncobj_destroy(screen->fd, syncobj->handle);
  85    free(syncobj);
  86 }
  87
  88 /**
  89  * Add a sync-point to the batch, with the given flags.
  90  *
  91  * \p flags   One of I915_EXEC_FENCE_WAIT or I915_EXEC_FENCE_SIGNAL.
  92  */
  93 void
  94 iris_batch_add_syncobj(struct iris_batch *batch,
  95                        struct iris_syncobj *syncobj,
  96                        unsigned flags)
  97 {
  98    struct drm_i915_gem_exec_fence *fence =
  99       util_dynarray_grow(&batch->exec_fences, struct drm_i915_gem_exec_fence, 1);
 100
 101    *fence = (struct drm_i915_gem_exec_fence) {
 102       .handle = syncobj->handle,
 103       .flags = flags,
 104    };
 105
 106    struct iris_syncobj **store =
 107       util_dynarray_grow(&batch->syncobjs, struct iris_syncobj *, 1);
 108
 109    *store = NULL;
 110    iris_syncobj_reference(batch->screen, store, syncobj);
 111 }
 112
 113 /* ------------------------------------------------------------------- */
 114
 115 struct pipe_fence_handle {
 116    struct pipe_reference ref;
 117    struct iris_seqno *seqno[IRIS_BATCH_COUNT];
 118 };
 119
 120 static void
 121 iris_fence_destroy(struct pipe_screen *p_screen,
 122                    struct pipe_fence_handle *fence)
 123 {
 124    struct iris_screen *screen = (struct iris_screen *)p_screen;
 125
 126    for (unsigned i = 0; i < ARRAY_SIZE(fence->seqno); i++)
 127       iris_seqno_reference(screen, &fence->seqno[i], NULL);
 128
 129    free(fence);
 130 }
 131
 132 static void
 133 iris_fence_reference(struct pipe_screen *p_screen,
 134                      struct pipe_fence_handle **dst,
 135                      struct pipe_fence_handle *src)
 136 {
 137    if (pipe_reference(*dst ? &(*dst)->ref : NULL,
 138                       src ? &src->ref : NULL))
 139       iris_fence_destroy(p_screen, *dst);
 140
 141    *dst = src;
 142 }
 143
 144 bool
 145 iris_wait_syncobj(struct pipe_screen *p_screen,
 146                   struct iris_syncobj *syncobj,
 147                   int64_t timeout_nsec)
 148 {
 149    if (!syncobj)
 150       return false;
 151
 152    struct iris_screen *screen = (struct iris_screen *)p_screen;
 153    struct drm_syncobj_wait args = {
 154       .handles = (uintptr_t)&syncobj->handle,
 155       .count_handles = 1,
 156       .timeout_nsec = timeout_nsec,
 157    };
 158    return gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args);
 159 }
 160
 161 #define CSI "\e["
 162 #define BLUE_HEADER  CSI "0;97;44m"
 163 #define NORMAL       CSI "0m"
 164
 165 static void
 166 iris_fence_flush(struct pipe_context *ctx,
 167                  struct pipe_fence_handle **out_fence,
 168                  unsigned flags)
 169 {
 170    struct iris_screen *screen = (void *) ctx->screen;
 171    struct iris_context *ice = (struct iris_context *)ctx;
 172
 173    if (flags & PIPE_FLUSH_END_OF_FRAME) {
 174       ice->frame++;
 175
 176       if (INTEL_DEBUG & DEBUG_SUBMIT) {
 177          fprintf(stderr, "%s ::: FRAME %-10u (ctx %p)%-35c%s\n",
 178                  (INTEL_DEBUG & DEBUG_COLOR) ? BLUE_HEADER : "",
 179                  ice->frame, ctx, ' ',
 180                  (INTEL_DEBUG & DEBUG_COLOR) ? NORMAL : "");
 181       }
 182    }
 183
 184    /* XXX PIPE_FLUSH_DEFERRED */
 185    for (unsigned i = 0; i < IRIS_BATCH_COUNT; i++)
 186       iris_batch_flush(&ice->batches[i]);
 187
 188    if (!out_fence)
 189       return;
 190
 191    struct pipe_fence_handle *fence = calloc(1, sizeof(*fence));
 192    if (!fence)
 193       return;
 194
 195    pipe_reference_init(&fence->ref, 1);
 196
 197    for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) {
 198       struct iris_batch *batch = &ice->batches[b];
 199
 200       if (iris_seqno_signaled(batch->last_seqno))
 201          continue;
 202
 203       iris_seqno_reference(screen, &fence->seqno[b], batch->last_seqno);
 204    }
 205
 206    iris_fence_reference(ctx->screen, out_fence, NULL);
 207    *out_fence = fence;
 208 }
 209
 210 static void
 211 iris_fence_await(struct pipe_context *ctx,
 212                  struct pipe_fence_handle *fence)
 213 {
 214    struct iris_context *ice = (struct iris_context *)ctx;
 215
 216    /* Flush any current work in our context as it doesn't need to wait
 217     * for this fence.  Any future work in our context must wait.
 218     */
 219    for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) {
 220       struct iris_batch *batch = &ice->batches[b];
 221
 222       for (unsigned i = 0; i < ARRAY_SIZE(fence->seqno); i++) {
 223          struct iris_seqno *seqno = fence->seqno[i];
 224
 225          if (iris_seqno_signaled(seqno))
 226             continue;
 227
 228          iris_batch_flush(batch);
 229          iris_batch_add_syncobj(batch, seqno->syncobj, I915_EXEC_FENCE_WAIT);
 230       }
 231    }
 232 }
 233
 234 #define NSEC_PER_SEC (1000 * USEC_PER_SEC)
 235 #define USEC_PER_SEC (1000 * MSEC_PER_SEC)
 236 #define MSEC_PER_SEC (1000)
 237
 238 static uint64_t
 239 gettime_ns(void)
 240 {
 241    struct timespec current;
 242    clock_gettime(CLOCK_MONOTONIC, &current);
 243    return (uint64_t)current.tv_sec * NSEC_PER_SEC + current.tv_nsec;
 244 }
 245
 246 static uint64_t
 247 rel2abs(uint64_t timeout)
 248 {
 249    if (timeout == 0)
 250       return 0;
 251
 252    uint64_t current_time = gettime_ns();
 253    uint64_t max_timeout = (uint64_t) INT64_MAX - current_time;
 254
 255    timeout = MIN2(max_timeout, timeout);
 256
 257    return current_time + timeout;
 258 }
 259
 260 static bool
 261 iris_fence_finish(struct pipe_screen *p_screen,
 262                   struct pipe_context *ctx,
 263                   struct pipe_fence_handle *fence,
 264                   uint64_t timeout)
 265 {
 266    struct iris_screen *screen = (struct iris_screen *)p_screen;
 267
 268    unsigned int handle_count = 0;
 269    uint32_t handles[ARRAY_SIZE(fence->seqno)];
 270    for (unsigned i = 0; i < ARRAY_SIZE(fence->seqno); i++) {
 271       struct iris_seqno *seqno = fence->seqno[i];
 272
 273       if (iris_seqno_signaled(seqno))
 274          continue;
 275
 276       handles[handle_count++] = seqno->syncobj->handle;
 277    }
 278
 279    if (handle_count == 0)
 280       return true;
 281
 282    struct drm_syncobj_wait args = {
 283       .handles = (uintptr_t)handles,
 284       .count_handles = handle_count,
 285       .timeout_nsec = rel2abs(timeout),
 286       .flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL
 287    };
 288    return gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args) == 0;
 289 }
 290
 291 static int
 292 sync_merge_fd(int sync_fd, int new_fd)
 293 {
 294    if (sync_fd == -1)
 295       return new_fd;
 296
 297    if (new_fd == -1)
 298       return sync_fd;
 299
 300    struct sync_merge_data args = {
 301       .name = "iris fence",
 302       .fd2 = new_fd,
 303       .fence = -1,
 304    };
 305
 306    gen_ioctl(sync_fd, SYNC_IOC_MERGE, &args);
 307    close(new_fd);
 308    close(sync_fd);
 309
 310    return args.fence;
 311 }
 312
 313 static int
 314 iris_fence_get_fd(struct pipe_screen *p_screen,
 315                   struct pipe_fence_handle *fence)
 316 {
 317    struct iris_screen *screen = (struct iris_screen *)p_screen;
 318    int fd = -1;
 319
 320    for (unsigned i = 0; i < ARRAY_SIZE(fence->seqno); i++) {
 321       struct iris_seqno *seqno = fence->seqno[i];
 322
 323       if (iris_seqno_signaled(seqno))
 324          continue;
 325
 326       struct drm_syncobj_handle args = {
 327          .handle = seqno->syncobj->handle,
 328          .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE,
 329          .fd = -1,
 330       };
 331
 332       gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
 333       fd = sync_merge_fd(fd, args.fd);
 334    }
 335
 336    if (fd == -1) {
 337       /* Our fence has no syncobj's recorded.  This means that all of the
 338        * batches had already completed, their syncobj's had been signalled,
 339        * and so we didn't bother to record them.  But we're being asked to
 340        * export such a fence.  So export a dummy already-signalled syncobj.
 341        */
 342       struct drm_syncobj_handle args = {
 343          .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE, .fd = -1,
 344       };
 345
 346       args.handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED);
 347       gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
 348       gem_syncobj_destroy(screen->fd, args.handle);
 349       return args.fd;
 350    }
 351
 352    return fd;
 353 }
 354
 355 static void
 356 iris_fence_create_fd(struct pipe_context *ctx,
 357                      struct pipe_fence_handle **out,
 358                      int fd,
 359                      enum pipe_fd_type type)
 360 {
 361    assert(type == PIPE_FD_TYPE_NATIVE_SYNC);
 362
 363    struct iris_screen *screen = (struct iris_screen *)ctx->screen;
 364    struct drm_syncobj_handle args = {
 365       .handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED),
 366       .flags = DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE,
 367       .fd = fd,
 368    };
 369    if (gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, &args) == -1) {
 370       fprintf(stderr, "DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE failed: %s\n",
 371               strerror(errno));
 372       gem_syncobj_destroy(screen->fd, args.handle);
 373       *out = NULL;
 374       return;
 375    }
 376
 377    struct iris_syncobj *syncobj = malloc(sizeof(*syncobj));
 378    if (!syncobj) {
 379       *out = NULL;
 380       return;
 381    }
 382    syncobj->handle = args.handle;
 383    pipe_reference_init(&syncobj->ref, 1);
 384
 385    struct iris_seqno *seqno = malloc(sizeof(*seqno));
 386    if (!seqno) {
 387       free(syncobj);
 388       *out = NULL;
 389       return;
 390    }
 391
 392    static const uint32_t zero = 0;
 393
 394    /* Fences work in terms of iris_seqno, but we don't actually have a
 395     * seqno for an imported fence.  So, create a fake one which always
 396     * returns as 'not signaled' so we fall back to using the sync object.
 397     */
 398    seqno->seqno = UINT32_MAX;
 399    seqno->map = &zero;
 400    seqno->syncobj = syncobj;
 401    seqno->flags = IRIS_SEQNO_END;
 402    pipe_reference_init(&seqno->reference, 1);
 403
 404    struct pipe_fence_handle *fence = calloc(1, sizeof(*fence));
 405    if (!fence) {
 406       free(seqno);
 407       free(syncobj);
 408       *out = NULL;
 409       return;
 410    }
 411    pipe_reference_init(&fence->ref, 1);
 412    fence->seqno[0] = seqno;
 413
 414    *out = fence;
 415 }
 416
 417 void
 418 iris_init_screen_fence_functions(struct pipe_screen *screen)
 419 {
 420    screen->fence_reference = iris_fence_reference;
 421    screen->fence_finish = iris_fence_finish;
 422    screen->fence_get_fd = iris_fence_get_fd;
 423 }
 424
 425 void
 426 iris_init_context_fence_functions(struct pipe_context *ctx)
 427 {
 428    ctx->flush = iris_fence_flush;
 429    ctx->create_fence_fd = iris_fence_create_fd;
 430    ctx->fence_server_sync = iris_fence_await;
 431 }