src/gallium/drivers/panfrost/pan_drm.c

   1 /*
   2  * © Copyright 2019 Collabora, Ltd.
   3  * Copyright 2019 Alyssa Rosenzweig
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  *
  24  */
  25
  26 #include <fcntl.h>
  27 #include <xf86drm.h>
  28
  29 #include "drm-uapi/panfrost_drm.h"
  30
  31 #include "util/u_memory.h"
  32 #include "util/os_time.h"
  33 #include "os/os_mman.h"
  34
  35 #include "pan_screen.h"
  36 #include "pan_resource.h"
  37 #include "pan_context.h"
  38 #include "pan_util.h"
  39 #include "pandecode/decode.h"
  40
  41 void
  42 panfrost_drm_mmap_bo(struct panfrost_screen *screen, struct panfrost_bo *bo)
  43 {
  44         struct drm_panfrost_mmap_bo mmap_bo = { .handle = bo->gem_handle };
  45         int ret;
  46
  47         if (bo->cpu)
  48                 return;
  49
  50         ret = drmIoctl(screen->fd, DRM_IOCTL_PANFROST_MMAP_BO, &mmap_bo);
  51         if (ret) {
  52                 fprintf(stderr, "DRM_IOCTL_PANFROST_MMAP_BO failed: %d\n", ret);
  53                 assert(0);
  54         }
  55
  56         bo->cpu = os_mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
  57                           screen->fd, mmap_bo.offset);
  58         if (bo->cpu == MAP_FAILED) {
  59                 fprintf(stderr, "mmap failed: %p\n", bo->cpu);
  60                 assert(0);
  61         }
  62
  63         /* Record the mmap if we're tracing */
  64         if (pan_debug & PAN_DBG_TRACE)
  65                 pandecode_inject_mmap(bo->gpu, bo->cpu, bo->size, NULL);
  66 }
  67
  68 static void
  69 panfrost_drm_munmap_bo(struct panfrost_screen *screen, struct panfrost_bo *bo)
  70 {
  71         if (!bo->cpu)
  72                 return;
  73
  74         if (os_munmap((void *) (uintptr_t)bo->cpu, bo->size)) {
  75                 perror("munmap");
  76                 abort();
  77         }
  78
  79         bo->cpu = NULL;
  80 }
  81
  82 struct panfrost_bo *
  83 panfrost_drm_create_bo(struct panfrost_screen *screen, size_t size,
  84                        uint32_t flags)
  85 {
  86         struct panfrost_bo *bo;
  87
  88         /* Kernel will fail (confusingly) with EPERM otherwise */
  89         assert(size > 0);
  90
  91         unsigned translated_flags = 0;
  92
  93         /* TODO: translate flags to kernel flags, if the kernel supports */
  94
  95         struct drm_panfrost_create_bo create_bo = {
  96                 .size = size,
  97                 .flags = translated_flags,
  98         };
  99
 100         /* Before creating a BO, we first want to check the cache */
 101
 102         bo = panfrost_bo_cache_fetch(screen, size, flags);
 103
 104         if (bo == NULL) {
 105                 /* Otherwise, the cache misses and we need to allocate a BO fresh from
 106                  * the kernel */
 107
 108                 int ret;
 109
 110                 ret = drmIoctl(screen->fd, DRM_IOCTL_PANFROST_CREATE_BO, &create_bo);
 111                 if (ret) {
 112                         fprintf(stderr, "DRM_IOCTL_PANFROST_CREATE_BO failed: %d\n", ret);
 113                         assert(0);
 114                 }
 115
 116                 /* We have a BO allocated from the kernel; fill in the userspace
 117                  * version */
 118
 119                 bo = rzalloc(screen, struct panfrost_bo);
 120                 bo->size = create_bo.size;
 121                 bo->gpu = create_bo.offset;
 122                 bo->gem_handle = create_bo.handle;
 123         }
 124
 125         /* Only mmap now if we know we need to. For CPU-invisible buffers, we
 126          * never map since we don't care about their contents; they're purely
 127          * for GPU-internal use. */
 128
 129         if (!(flags & (PAN_ALLOCATE_INVISIBLE | PAN_ALLOCATE_DELAY_MMAP)))
 130                 panfrost_drm_mmap_bo(screen, bo);
 131
 132         pipe_reference_init(&bo->reference, 1);
 133         return bo;
 134 }
 135
 136 void
 137 panfrost_drm_release_bo(struct panfrost_screen *screen, struct panfrost_bo *bo)
 138 {
 139         struct drm_gem_close gem_close = { .handle = bo->gem_handle };
 140         int ret;
 141
 142         if (!bo)
 143                 return;
 144
 145         panfrost_drm_munmap_bo(screen, bo);
 146
 147         ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &gem_close);
 148         if (ret) {
 149                 fprintf(stderr, "DRM_IOCTL_GEM_CLOSE failed: %d\n", ret);
 150                 assert(0);
 151         }
 152
 153         ralloc_free(bo);
 154 }
 155
 156 void
 157 panfrost_drm_allocate_slab(struct panfrost_screen *screen,
 158                            struct panfrost_memory *mem,
 159                            size_t pages,
 160                            bool same_va,
 161                            int extra_flags,
 162                            int commit_count,
 163                            int extent)
 164 {
 165         // TODO cache allocations
 166         // TODO properly handle errors
 167         // TODO take into account extra_flags
 168         mem->bo = panfrost_drm_create_bo(screen, pages * 4096, extra_flags);
 169         mem->stack_bottom = 0;
 170 }
 171
 172 void
 173 panfrost_drm_free_slab(struct panfrost_screen *screen, struct panfrost_memory *mem)
 174 {
 175         panfrost_bo_unreference(&screen->base, mem->bo);
 176         mem->bo = NULL;
 177 }
 178
 179 struct panfrost_bo *
 180 panfrost_drm_import_bo(struct panfrost_screen *screen, int fd)
 181 {
 182         struct panfrost_bo *bo = rzalloc(screen, struct panfrost_bo);
 183         struct drm_panfrost_get_bo_offset get_bo_offset = {0,};
 184         MAYBE_UNUSED int ret;
 185         unsigned gem_handle;
 186
 187         ret = drmPrimeFDToHandle(screen->fd, fd, &gem_handle);
 188         assert(!ret);
 189
 190         get_bo_offset.handle = gem_handle;
 191         ret = drmIoctl(screen->fd, DRM_IOCTL_PANFROST_GET_BO_OFFSET, &get_bo_offset);
 192         assert(!ret);
 193
 194         bo->gem_handle = gem_handle;
 195         bo->gpu = (mali_ptr) get_bo_offset.offset;
 196         bo->size = lseek(fd, 0, SEEK_END);
 197         assert(bo->size > 0);
 198         pipe_reference_init(&bo->reference, 1);
 199
 200         // TODO map and unmap on demand?
 201         panfrost_drm_mmap_bo(screen, bo);
 202         return bo;
 203 }
 204
 205 int
 206 panfrost_drm_export_bo(struct panfrost_screen *screen, const struct panfrost_bo *bo)
 207 {
 208         struct drm_prime_handle args = {
 209                 .handle = bo->gem_handle,
 210                 .flags = DRM_CLOEXEC,
 211         };
 212
 213         int ret = drmIoctl(screen->fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &args);
 214         if (ret == -1)
 215                 return -1;
 216
 217         return args.fd;
 218 }
 219
 220 static int
 221 panfrost_drm_submit_job(struct panfrost_context *ctx, u64 job_desc, int reqs)
 222 {
 223         struct pipe_context *gallium = (struct pipe_context *) ctx;
 224         struct panfrost_screen *screen = pan_screen(gallium->screen);
 225         struct panfrost_job *job = panfrost_get_job_for_fbo(ctx);
 226         struct drm_panfrost_submit submit = {0,};
 227         int *bo_handles, ret;
 228
 229         submit.in_syncs = (u64) (uintptr_t) &ctx->out_sync;
 230         submit.in_sync_count = 1;
 231
 232         submit.out_sync = ctx->out_sync;
 233
 234         submit.jc = job_desc;
 235         submit.requirements = reqs;
 236
 237         bo_handles = calloc(job->bos->entries, sizeof(*bo_handles));
 238         assert(bo_handles);
 239
 240         set_foreach(job->bos, entry) {
 241                 struct panfrost_bo *bo = (struct panfrost_bo *)entry->key;
 242                 assert(bo->gem_handle > 0);
 243                 bo_handles[submit.bo_handle_count++] = bo->gem_handle;
 244         }
 245
 246         submit.bo_handles = (u64) (uintptr_t) bo_handles;
 247         ret = drmIoctl(screen->fd, DRM_IOCTL_PANFROST_SUBMIT, &submit);
 248         free(bo_handles);
 249         if (ret) {
 250                 fprintf(stderr, "Error submitting: %m\n");
 251                 return errno;
 252         }
 253
 254         /* Trace the job if we're doing that */
 255         if (pan_debug & PAN_DBG_TRACE) {
 256                 /* Wait so we can get errors reported back */
 257                 drmSyncobjWait(screen->fd, &ctx->out_sync, 1, INT64_MAX, 0, NULL);
 258                 pandecode_jc(submit.jc, FALSE);
 259         }
 260
 261         return 0;
 262 }
 263
 264 int
 265 panfrost_drm_submit_vs_fs_job(struct panfrost_context *ctx, bool has_draws, bool is_scanout)
 266 {
 267         int ret = 0;
 268
 269         struct panfrost_job *job = panfrost_get_job_for_fbo(ctx);
 270
 271         /* TODO: Add here the transient pools */
 272         panfrost_job_add_bo(job, ctx->shaders.bo);
 273         panfrost_job_add_bo(job, ctx->scratchpad.bo);
 274         panfrost_job_add_bo(job, ctx->tiler_heap.bo);
 275         panfrost_job_add_bo(job, ctx->varying_mem.bo);
 276         panfrost_job_add_bo(job, ctx->tiler_polygon_list.bo);
 277
 278         if (job->first_job.gpu) {
 279                 ret = panfrost_drm_submit_job(ctx, job->first_job.gpu, 0);
 280                 assert(!ret);
 281         }
 282
 283         if (job->first_tiler.gpu || job->clear) {
 284                 struct pipe_surface *surf = ctx->pipe_framebuffer.cbufs[0];
 285                 if (surf) {
 286                         struct panfrost_resource *res = pan_resource(surf->texture);
 287                         assert(res->bo);
 288                         panfrost_job_add_bo(job, res->bo);
 289                 }
 290                 ret = panfrost_drm_submit_job(ctx, panfrost_fragment_job(ctx, has_draws), PANFROST_JD_REQ_FS);
 291                 assert(!ret);
 292         }
 293
 294         return ret;
 295 }
 296
 297 static struct panfrost_fence *
 298 panfrost_fence_create(struct panfrost_context *ctx)
 299 {
 300         struct pipe_context *gallium = (struct pipe_context *) ctx;
 301         struct panfrost_screen *screen = pan_screen(gallium->screen);
 302         struct panfrost_fence *f = calloc(1, sizeof(*f));
 303         if (!f)
 304                 return NULL;
 305
 306         /* Snapshot the last Panfrost's rendering's out fence.  We'd rather have
 307          * another syncobj instead of a sync file, but this is all we get.
 308          * (HandleToFD/FDToHandle just gives you another syncobj ID for the
 309          * same syncobj).
 310          */
 311         drmSyncobjExportSyncFile(screen->fd, ctx->out_sync, &f->fd);
 312         if (f->fd == -1) {
 313                 fprintf(stderr, "export failed\n");
 314                 free(f);
 315                 return NULL;
 316         }
 317
 318         pipe_reference_init(&f->reference, 1);
 319
 320         return f;
 321 }
 322
 323 void
 324 panfrost_drm_force_flush_fragment(struct panfrost_context *ctx,
 325                                   struct pipe_fence_handle **fence)
 326 {
 327         struct pipe_context *gallium = (struct pipe_context *) ctx;
 328         struct panfrost_screen *screen = pan_screen(gallium->screen);
 329
 330         if (!screen->last_fragment_flushed) {
 331                 drmSyncobjWait(screen->fd, &ctx->out_sync, 1, INT64_MAX, 0, NULL);
 332                 screen->last_fragment_flushed = true;
 333
 334                 /* The job finished up, so we're safe to clean it up now */
 335                 panfrost_free_job(ctx, screen->last_job);
 336         }
 337
 338         if (fence) {
 339                 struct panfrost_fence *f = panfrost_fence_create(ctx);
 340                 gallium->screen->fence_reference(gallium->screen, fence, NULL);
 341                 *fence = (struct pipe_fence_handle *)f;
 342         }
 343 }
 344
 345 unsigned
 346 panfrost_drm_query_gpu_version(struct panfrost_screen *screen)
 347 {
 348         struct drm_panfrost_get_param get_param = {0,};
 349         MAYBE_UNUSED int ret;
 350
 351         get_param.param = DRM_PANFROST_PARAM_GPU_PROD_ID;
 352         ret = drmIoctl(screen->fd, DRM_IOCTL_PANFROST_GET_PARAM, &get_param);
 353         assert(!ret);
 354
 355         return get_param.value;
 356 }
 357
 358 int
 359 panfrost_drm_init_context(struct panfrost_context *ctx)
 360 {
 361         struct pipe_context *gallium = (struct pipe_context *) ctx;
 362         struct panfrost_screen *screen = pan_screen(gallium->screen);
 363
 364         return drmSyncobjCreate(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED,
 365                                 &ctx->out_sync);
 366 }
 367
 368 void
 369 panfrost_drm_fence_reference(struct pipe_screen *screen,
 370                              struct pipe_fence_handle **ptr,
 371                              struct pipe_fence_handle *fence)
 372 {
 373         struct panfrost_fence **p = (struct panfrost_fence **)ptr;
 374         struct panfrost_fence *f = (struct panfrost_fence *)fence;
 375         struct panfrost_fence *old = *p;
 376
 377         if (pipe_reference(&(*p)->reference, &f->reference)) {
 378                 close(old->fd);
 379                 free(old);
 380         }
 381         *p = f;
 382 }
 383
 384 boolean
 385 panfrost_drm_fence_finish(struct pipe_screen *pscreen,
 386                           struct pipe_context *ctx,
 387                           struct pipe_fence_handle *fence,
 388                           uint64_t timeout)
 389 {
 390         struct panfrost_screen *screen = pan_screen(pscreen);
 391         struct panfrost_fence *f = (struct panfrost_fence *)fence;
 392         int ret;
 393
 394         unsigned syncobj;
 395         ret = drmSyncobjCreate(screen->fd, 0, &syncobj);
 396         if (ret) {
 397                 fprintf(stderr, "Failed to create syncobj to wait on: %m\n");
 398                 return false;
 399         }
 400
 401         drmSyncobjImportSyncFile(screen->fd, syncobj, f->fd);
 402         if (ret) {
 403                 fprintf(stderr, "Failed to import fence to syncobj: %m\n");
 404                 return false;
 405         }
 406
 407         uint64_t abs_timeout = os_time_get_absolute_timeout(timeout);
 408         if (abs_timeout == OS_TIMEOUT_INFINITE)
 409                 abs_timeout = INT64_MAX;
 410
 411         ret = drmSyncobjWait(screen->fd, &syncobj, 1, abs_timeout, 0, NULL);
 412
 413         drmSyncobjDestroy(screen->fd, syncobj);
 414
 415         return ret >= 0;
 416 }