src/gallium/drivers/iris/iris_fence.c

   1 /*
   2  * Copyright © 2018 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20  * DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 /**
  24  * @file iris_fence.c
  25  *
  26  * Fences for driver and IPC serialisation, scheduling and synchronisation.
  27  */
  28
  29 #include "util/u_inlines.h"
  30 #include "intel/common/gen_gem.h"
  31
  32 #include "iris_batch.h"
  33 #include "iris_bufmgr.h"
  34 #include "iris_context.h"
  35 #include "iris_fence.h"
  36 #include "iris_screen.h"
  37
  38 static uint32_t
  39 gem_syncobj_create(int fd, uint32_t flags)
  40 {
  41    struct drm_syncobj_create args = {
  42       .flags = flags,
  43    };
  44
  45    gen_ioctl(fd, DRM_IOCTL_SYNCOBJ_CREATE, &args);
  46
  47    return args.handle;
  48 }
  49
  50 static void
  51 gem_syncobj_destroy(int fd, uint32_t handle)
  52 {
  53    struct drm_syncobj_destroy args = {
  54       .handle = handle,
  55    };
  56
  57    gen_ioctl(fd, DRM_IOCTL_SYNCOBJ_DESTROY, &args);
  58 }
  59
  60 /**
  61  * Make a new sync-point.
  62  */
  63 struct iris_syncpt *
  64 iris_create_syncpt(struct iris_screen *screen)
  65 {
  66    struct iris_syncpt *syncpt = malloc(sizeof(*syncpt));
  67
  68    if (!syncpt)
  69       return NULL;
  70
  71    syncpt->handle = gem_syncobj_create(screen->fd, 0);
  72    assert(syncpt->handle);
  73
  74    pipe_reference_init(&syncpt->ref, 1);
  75
  76    return syncpt;
  77 }
  78
  79 void
  80 iris_syncpt_destroy(struct iris_screen *screen, struct iris_syncpt *syncpt)
  81 {
  82    gem_syncobj_destroy(screen->fd, syncpt->handle);
  83    free(syncpt);
  84 }
  85
  86 /**
  87  * Add a sync-point to the batch, with the given flags.
  88  *
  89  * \p flags   One of I915_EXEC_FENCE_WAIT or I915_EXEC_FENCE_SIGNAL.
  90  */
  91 void
  92 iris_batch_add_syncpt(struct iris_batch *batch,
  93                       struct iris_syncpt *syncpt,
  94                       unsigned flags)
  95 {
  96    struct drm_i915_gem_exec_fence *fence =
  97       util_dynarray_grow(&batch->exec_fences, struct drm_i915_gem_exec_fence, 1);
  98
  99    *fence = (struct drm_i915_gem_exec_fence) {
 100       .handle = syncpt->handle,
 101       .flags = flags,
 102    };
 103
 104    struct iris_syncpt **store =
 105       util_dynarray_grow(&batch->syncpts, struct iris_syncpt *, 1);
 106
 107    *store = NULL;
 108    iris_syncpt_reference(batch->screen, store, syncpt);
 109 }
 110
 111 /* ------------------------------------------------------------------- */
 112
 113 struct pipe_fence_handle {
 114    struct pipe_reference ref;
 115    struct iris_syncpt *syncpt[IRIS_BATCH_COUNT];
 116    unsigned count;
 117 };
 118
 119 static void
 120 iris_fence_destroy(struct pipe_screen *p_screen,
 121                    struct pipe_fence_handle *fence)
 122 {
 123    struct iris_screen *screen = (struct iris_screen *)p_screen;
 124
 125    for (unsigned i = 0; i < fence->count; i++)
 126       iris_syncpt_reference(screen, &fence->syncpt[i], NULL);
 127
 128    free(fence);
 129 }
 130
 131 static void
 132 iris_fence_reference(struct pipe_screen *p_screen,
 133                      struct pipe_fence_handle **dst,
 134                      struct pipe_fence_handle *src)
 135 {
 136    if (pipe_reference(*dst ? &(*dst)->ref : NULL,
 137                       src ? &src->ref : NULL))
 138       iris_fence_destroy(p_screen, *dst);
 139
 140    *dst = src;
 141 }
 142
 143 bool
 144 iris_wait_syncpt(struct pipe_screen *p_screen,
 145                  struct iris_syncpt *syncpt,
 146                  int64_t timeout_nsec)
 147 {
 148    if (!syncpt)
 149       return false;
 150
 151    struct iris_screen *screen = (struct iris_screen *)p_screen;
 152    struct drm_syncobj_wait args = {
 153       .handles = (uintptr_t)&syncpt->handle,
 154       .count_handles = 1,
 155       .timeout_nsec = timeout_nsec,
 156    };
 157    return gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args);
 158 }
 159
 160 #define CSI "\e["
 161 #define BLUE_HEADER  CSI "0;97;44m"
 162 #define NORMAL       CSI "0m"
 163
 164 static void
 165 iris_fence_flush(struct pipe_context *ctx,
 166                  struct pipe_fence_handle **out_fence,
 167                  unsigned flags)
 168 {
 169    struct iris_screen *screen = (void *) ctx->screen;
 170    struct iris_context *ice = (struct iris_context *)ctx;
 171
 172    if (flags & PIPE_FLUSH_END_OF_FRAME) {
 173       ice->frame++;
 174
 175       if (INTEL_DEBUG & DEBUG_SUBMIT) {
 176          fprintf(stderr, "%s ::: FRAME %-10u (ctx %p)%-35c%s\n",
 177                  (INTEL_DEBUG & DEBUG_COLOR) ? BLUE_HEADER : "",
 178                  ice->frame, ctx, ' ',
 179                  (INTEL_DEBUG & DEBUG_COLOR) ? NORMAL : "");
 180       }
 181    }
 182
 183    /* XXX PIPE_FLUSH_DEFERRED */
 184    for (unsigned i = 0; i < IRIS_BATCH_COUNT; i++)
 185       iris_batch_flush(&ice->batches[i]);
 186
 187    if (!out_fence)
 188       return;
 189
 190    struct pipe_fence_handle *fence = calloc(1, sizeof(*fence));
 191    if (!fence)
 192       return;
 193
 194    pipe_reference_init(&fence->ref, 1);
 195
 196    for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) {
 197       if (!iris_wait_syncpt(ctx->screen, ice->batches[b].last_syncpt, 0))
 198          continue;
 199
 200       iris_syncpt_reference(screen, &fence->syncpt[fence->count++],
 201                             ice->batches[b].last_syncpt);
 202    }
 203
 204    iris_fence_reference(ctx->screen, out_fence, NULL);
 205    *out_fence = fence;
 206 }
 207
 208 static void
 209 iris_fence_await(struct pipe_context *ctx,
 210                  struct pipe_fence_handle *fence)
 211 {
 212    struct iris_context *ice = (struct iris_context *)ctx;
 213
 214    for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) {
 215       for (unsigned i = 0; i < fence->count; i++) {
 216          iris_batch_add_syncpt(&ice->batches[b], fence->syncpt[i],
 217                                I915_EXEC_FENCE_WAIT);
 218       }
 219    }
 220 }
 221
 222 #define NSEC_PER_SEC (1000 * USEC_PER_SEC)
 223 #define USEC_PER_SEC (1000 * MSEC_PER_SEC)
 224 #define MSEC_PER_SEC (1000)
 225
 226 static uint64_t
 227 gettime_ns(void)
 228 {
 229    struct timespec current;
 230    clock_gettime(CLOCK_MONOTONIC, &current);
 231    return (uint64_t)current.tv_sec * NSEC_PER_SEC + current.tv_nsec;
 232 }
 233
 234 static uint64_t
 235 rel2abs(uint64_t timeout)
 236 {
 237    if (timeout == 0)
 238       return 0;
 239
 240    uint64_t current_time = gettime_ns();
 241    uint64_t max_timeout = (uint64_t) INT64_MAX - current_time;
 242
 243    timeout = MIN2(max_timeout, timeout);
 244
 245    return current_time + timeout;
 246 }
 247
 248 static bool
 249 iris_fence_finish(struct pipe_screen *p_screen,
 250                   struct pipe_context *ctx,
 251                   struct pipe_fence_handle *fence,
 252                   uint64_t timeout)
 253 {
 254    struct iris_screen *screen = (struct iris_screen *)p_screen;
 255
 256    if (!fence->count)
 257       return true;
 258
 259    uint32_t handles[ARRAY_SIZE(fence->syncpt)];
 260    for (unsigned i = 0; i < fence->count; i++)
 261       handles[i] = fence->syncpt[i]->handle;
 262
 263    struct drm_syncobj_wait args = {
 264       .handles = (uintptr_t)handles,
 265       .count_handles = fence->count,
 266       .timeout_nsec = rel2abs(timeout),
 267       .flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL
 268    };
 269    return gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args) == 0;
 270 }
 271
 272 #ifndef SYNC_IOC_MAGIC
 273 /* duplicated from linux/sync_file.h to avoid build-time dependency
 274  * on new (v4.7) kernel headers.  Once distro's are mostly using
 275  * something newer than v4.7 drop this and #include <linux/sync_file.h>
 276  * instead.
 277  */
 278 struct sync_merge_data {
 279    char  name[32];
 280    __s32 fd2;
 281    __s32 fence;
 282    __u32 flags;
 283    __u32 pad;
 284 };
 285
 286 #define SYNC_IOC_MAGIC '>'
 287 #define SYNC_IOC_MERGE _IOWR(SYNC_IOC_MAGIC, 3, struct sync_merge_data)
 288 #endif
 289
 290 static int
 291 sync_merge_fd(int sync_fd, int new_fd)
 292 {
 293    if (sync_fd == -1)
 294       return new_fd;
 295
 296    if (new_fd == -1)
 297       return sync_fd;
 298
 299    struct sync_merge_data args = {
 300       .name = "iris fence",
 301       .fd2 = new_fd,
 302       .fence = -1,
 303    };
 304
 305    gen_ioctl(sync_fd, SYNC_IOC_MERGE, &args);
 306    close(new_fd);
 307    close(sync_fd);
 308
 309    return args.fence;
 310 }
 311
 312 static int
 313 iris_fence_get_fd(struct pipe_screen *p_screen,
 314                   struct pipe_fence_handle *fence)
 315 {
 316    struct iris_screen *screen = (struct iris_screen *)p_screen;
 317    int fd = -1;
 318
 319    if (fence->count == 0) {
 320       /* Our fence has no syncobj's recorded.  This means that all of the
 321        * batches had already completed, their syncobj's had been signalled,
 322        * and so we didn't bother to record them.  But we're being asked to
 323        * export such a fence.  So export a dummy already-signalled syncobj.
 324        */
 325       struct drm_syncobj_handle args = {
 326          .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE, .fd = -1,
 327       };
 328
 329       args.handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED);
 330       gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
 331       gem_syncobj_destroy(screen->fd, args.handle);
 332       return args.fd;
 333    }
 334
 335    for (unsigned i = 0; i < fence->count; i++) {
 336       struct drm_syncobj_handle args = {
 337          .handle = fence->syncpt[i]->handle,
 338          .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE,
 339          .fd = -1,
 340       };
 341
 342       gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
 343       fd = sync_merge_fd(fd, args.fd);
 344    }
 345
 346    return fd;
 347 }
 348
 349 static void
 350 iris_fence_create_fd(struct pipe_context *ctx,
 351                      struct pipe_fence_handle **out,
 352                      int fd,
 353                      enum pipe_fd_type type)
 354 {
 355    assert(type == PIPE_FD_TYPE_NATIVE_SYNC);
 356
 357    struct iris_screen *screen = (struct iris_screen *)ctx->screen;
 358    struct drm_syncobj_handle args = {
 359       .handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED),
 360       .flags = DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE,
 361       .fd = fd,
 362    };
 363    if (gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, &args) == -1) {
 364       fprintf(stderr, "DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE failed: %s\n",
 365               strerror(errno));
 366       gem_syncobj_destroy(screen->fd, args.handle);
 367       *out = NULL;
 368       return;
 369    }
 370
 371    struct iris_syncpt *syncpt = malloc(sizeof(*syncpt));
 372    syncpt->handle = args.handle;
 373    pipe_reference_init(&syncpt->ref, 1);
 374
 375    struct pipe_fence_handle *fence = malloc(sizeof(*fence));
 376    pipe_reference_init(&fence->ref, 1);
 377    fence->syncpt[0] = syncpt;
 378    fence->count = 1;
 379
 380    *out = fence;
 381 }
 382
 383 void
 384 iris_init_screen_fence_functions(struct pipe_screen *screen)
 385 {
 386    screen->fence_reference = iris_fence_reference;
 387    screen->fence_finish = iris_fence_finish;
 388    screen->fence_get_fd = iris_fence_get_fd;
 389 }
 390
 391 void
 392 iris_init_context_fence_functions(struct pipe_context *ctx)
 393 {
 394    ctx->flush = iris_fence_flush;
 395    ctx->create_fence_fd = iris_fence_create_fd;
 396    ctx->fence_server_sync = iris_fence_await;
 397 }