src/gallium/drivers/lima/lima_bo.c

   1 /*
   2  * Copyright (C) 2017-2019 Lima Project
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <stdlib.h>
  25 #include <sys/types.h>
  26 #include <unistd.h>
  27 #include <fcntl.h>
  28
  29 #include "xf86drm.h"
  30 #include "drm-uapi/lima_drm.h"
  31
  32 #include "util/u_hash_table.h"
  33 #include "util/u_math.h"
  34 #include "util/os_time.h"
  35 #include "os/os_mman.h"
  36
  37 #include "state_tracker/drm_driver.h"
  38
  39 #include "lima_screen.h"
  40 #include "lima_bo.h"
  41 #include "lima_util.h"
  42
  43 #define PTR_TO_UINT(x) ((unsigned)((intptr_t)(x)))
  44
  45 static unsigned handle_hash(void *key)
  46 {
  47     return PTR_TO_UINT(key);
  48 }
  49
  50 static int handle_compare(void *key1, void *key2)
  51 {
  52     return PTR_TO_UINT(key1) != PTR_TO_UINT(key2);
  53 }
  54
  55 bool lima_bo_table_init(struct lima_screen *screen)
  56 {
  57    screen->bo_handles = util_hash_table_create(handle_hash, handle_compare);
  58    if (!screen->bo_handles)
  59       return false;
  60
  61    screen->bo_flink_names = util_hash_table_create(handle_hash, handle_compare);
  62    if (!screen->bo_flink_names)
  63       goto err_out0;
  64
  65    mtx_init(&screen->bo_table_lock, mtx_plain);
  66    return true;
  67
  68 err_out0:
  69    util_hash_table_destroy(screen->bo_handles);
  70    return false;
  71 }
  72
  73 bool lima_bo_cache_init(struct lima_screen *screen)
  74 {
  75    mtx_init(&screen->bo_cache_lock, mtx_plain);
  76    list_inithead(&screen->bo_cache_time);
  77    for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++)
  78       list_inithead(&screen->bo_cache_buckets[i]);
  79
  80    return true;
  81 }
  82
  83 void lima_bo_table_fini(struct lima_screen *screen)
  84 {
  85    mtx_destroy(&screen->bo_table_lock);
  86    util_hash_table_destroy(screen->bo_handles);
  87    util_hash_table_destroy(screen->bo_flink_names);
  88 }
  89
  90 static void
  91 lima_bo_cache_remove(struct lima_bo *bo)
  92 {
  93    list_del(&bo->size_list);
  94    list_del(&bo->time_list);
  95 }
  96
  97 static void lima_close_kms_handle(struct lima_screen *screen, uint32_t handle)
  98 {
  99    struct drm_gem_close args = {
 100       .handle = handle,
 101    };
 102
 103    drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &args);
 104 }
 105
 106 static void
 107 lima_bo_free(struct lima_bo *bo)
 108 {
 109    struct lima_screen *screen = bo->screen;
 110
 111    if (lima_debug & LIMA_DEBUG_BO_CACHE)
 112       fprintf(stderr, "%s: %p (size=%d)\n", __func__,
 113               bo, bo->size);
 114
 115    mtx_lock(&screen->bo_table_lock);
 116    util_hash_table_remove(screen->bo_handles,
 117                           (void *)(uintptr_t)bo->handle);
 118    if (bo->flink_name)
 119       util_hash_table_remove(screen->bo_flink_names,
 120                              (void *)(uintptr_t)bo->flink_name);
 121    mtx_unlock(&screen->bo_table_lock);
 122
 123    if (bo->map)
 124       lima_bo_unmap(bo);
 125
 126    lima_close_kms_handle(screen, bo->handle);
 127    free(bo);
 128 }
 129
 130 void lima_bo_cache_fini(struct lima_screen *screen)
 131 {
 132    mtx_destroy(&screen->bo_cache_lock);
 133
 134    list_for_each_entry_safe(struct lima_bo, entry,
 135                             &screen->bo_cache_time, time_list) {
 136       lima_bo_cache_remove(entry);
 137       lima_bo_free(entry);
 138    }
 139 }
 140
 141 static bool lima_bo_get_info(struct lima_bo *bo)
 142 {
 143    struct drm_lima_gem_info req = {
 144       .handle = bo->handle,
 145    };
 146
 147    if(drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_INFO, &req))
 148       return false;
 149
 150    bo->offset = req.offset;
 151    bo->va = req.va;
 152    return true;
 153 }
 154
 155 static unsigned
 156 lima_bucket_index(unsigned size)
 157 {
 158    /* Round down to POT to compute a bucket index */
 159
 160    unsigned bucket_index = util_logbase2(size);
 161
 162    /* Clamp the bucket index; all huge allocations will be
 163     * sorted into the largest bucket */
 164    bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET,
 165                         MAX_BO_CACHE_BUCKET);
 166
 167    /* Reindex from 0 */
 168    return (bucket_index - MIN_BO_CACHE_BUCKET);
 169 }
 170
 171 static struct list_head *
 172 lima_bo_cache_get_bucket(struct lima_screen *screen, unsigned size)
 173 {
 174    return &screen->bo_cache_buckets[lima_bucket_index(size)];
 175 }
 176
 177 static void
 178 lima_bo_cache_free_stale_bos(struct lima_screen *screen, time_t time)
 179 {
 180    unsigned cnt = 0;
 181    list_for_each_entry_safe(struct lima_bo, entry,
 182                             &screen->bo_cache_time, time_list) {
 183       /* Free BOs that are sitting idle for longer than 5 seconds */
 184       if (time - entry->free_time > 6) {
 185          lima_bo_cache_remove(entry);
 186          lima_bo_free(entry);
 187          cnt++;
 188       } else
 189          break;
 190    }
 191    if ((lima_debug & LIMA_DEBUG_BO_CACHE) && cnt)
 192       fprintf(stderr, "%s: freed %d stale BOs\n", __func__, cnt);
 193 }
 194
 195 static void
 196 lima_bo_cache_print_stats(struct lima_screen *screen)
 197 {
 198    fprintf(stderr, "===============\n");
 199    fprintf(stderr, "BO cache stats:\n");
 200    unsigned total_size = 0;
 201    for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++) {
 202       struct list_head *bucket = &screen->bo_cache_buckets[i];
 203       unsigned bucket_size = 0;
 204       list_for_each_entry(struct lima_bo, entry, bucket, size_list) {
 205          bucket_size += entry->size;
 206          total_size += entry->size;
 207       }
 208       fprintf(stderr, "Bucket #%d, BOs: %d, size: %u\n", i,
 209               list_length(bucket),
 210               bucket_size);
 211    }
 212    fprintf(stderr, "Total size: %u\n", total_size);
 213 }
 214
 215 static bool
 216 lima_bo_cache_put(struct lima_bo *bo)
 217 {
 218    if (!bo->cacheable)
 219       return false;
 220
 221    struct lima_screen *screen = bo->screen;
 222
 223    mtx_lock(&screen->bo_cache_lock);
 224    struct list_head *bucket = lima_bo_cache_get_bucket(screen, bo->size);
 225
 226    if (!bucket) {
 227       mtx_unlock(&screen->bo_cache_lock);
 228       return false;
 229    }
 230
 231    struct timespec time;
 232    clock_gettime(CLOCK_MONOTONIC, &time);
 233    bo->free_time = time.tv_sec;
 234    list_addtail(&bo->size_list, bucket);
 235    list_addtail(&bo->time_list, &screen->bo_cache_time);
 236    lima_bo_cache_free_stale_bos(screen, time.tv_sec);
 237    if (lima_debug & LIMA_DEBUG_BO_CACHE) {
 238       fprintf(stderr, "%s: put BO: %p (size=%d)\n", __func__, bo, bo->size);
 239       lima_bo_cache_print_stats(screen);
 240    }
 241    mtx_unlock(&screen->bo_cache_lock);
 242
 243    return true;
 244 }
 245
 246 static struct lima_bo *
 247 lima_bo_cache_get(struct lima_screen *screen, uint32_t size, uint32_t flags)
 248 {
 249    struct lima_bo *bo = NULL;
 250    mtx_lock(&screen->bo_cache_lock);
 251    struct list_head *bucket = lima_bo_cache_get_bucket(screen, size);
 252
 253    if (!bucket) {
 254       mtx_unlock(&screen->bo_cache_lock);
 255       return false;
 256    }
 257
 258    list_for_each_entry_safe(struct lima_bo, entry, bucket, size_list) {
 259       if (entry->size >= size) {
 260          /* Check if BO is idle. If it's not it's better to allocate new one */
 261          if (!lima_bo_wait(entry, LIMA_GEM_WAIT_WRITE, 0)) {
 262             if (lima_debug & LIMA_DEBUG_BO_CACHE) {
 263                fprintf(stderr, "%s: found BO %p but it's busy\n", __func__,
 264                        entry);
 265             }
 266             break;
 267          }
 268
 269          lima_bo_cache_remove(entry);
 270          p_atomic_set(&entry->refcnt, 1);
 271          entry->flags = flags;
 272          bo = entry;
 273          if (lima_debug & LIMA_DEBUG_BO_CACHE) {
 274             fprintf(stderr, "%s: got BO: %p (size=%d), requested size %d\n",
 275                     __func__, bo, bo->size, size);
 276             lima_bo_cache_print_stats(screen);
 277          }
 278          break;
 279       }
 280    }
 281
 282    mtx_unlock(&screen->bo_cache_lock);
 283
 284    return bo;
 285 }
 286
 287 struct lima_bo *lima_bo_create(struct lima_screen *screen,
 288                                uint32_t size, uint32_t flags)
 289 {
 290    struct lima_bo *bo;
 291
 292    size = align(size, LIMA_PAGE_SIZE);
 293
 294    /* Try to get bo from cache first */
 295    bo = lima_bo_cache_get(screen, size, flags);
 296    if (bo)
 297       return bo;
 298
 299    struct drm_lima_gem_create req = {
 300       .size = size,
 301       .flags = flags,
 302    };
 303
 304    if (!(bo = calloc(1, sizeof(*bo))))
 305       return NULL;
 306
 307    list_inithead(&bo->time_list);
 308    list_inithead(&bo->size_list);
 309
 310    if (drmIoctl(screen->fd, DRM_IOCTL_LIMA_GEM_CREATE, &req))
 311       goto err_out0;
 312
 313    bo->screen = screen;
 314    bo->size = req.size;
 315    bo->flags = req.flags;
 316    bo->handle = req.handle;
 317    bo->cacheable = !(lima_debug & LIMA_DEBUG_NO_BO_CACHE);
 318    p_atomic_set(&bo->refcnt, 1);
 319
 320    if (!lima_bo_get_info(bo))
 321       goto err_out1;
 322
 323    if (lima_debug & LIMA_DEBUG_BO_CACHE)
 324       fprintf(stderr, "%s: %p (size=%d)\n", __func__,
 325               bo, bo->size);
 326
 327    return bo;
 328
 329 err_out1:
 330    lima_close_kms_handle(screen, bo->handle);
 331 err_out0:
 332    free(bo);
 333    return NULL;
 334 }
 335
 336 void lima_bo_unreference(struct lima_bo *bo)
 337 {
 338    if (!p_atomic_dec_zero(&bo->refcnt))
 339       return;
 340
 341    /* Try to put it into cache */
 342    if (lima_bo_cache_put(bo))
 343       return;
 344
 345    lima_bo_free(bo);
 346 }
 347
 348 void *lima_bo_map(struct lima_bo *bo)
 349 {
 350    if (!bo->map) {
 351       bo->map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE,
 352                         MAP_SHARED, bo->screen->fd, bo->offset);
 353       if (bo->map == MAP_FAILED)
 354           bo->map = NULL;
 355    }
 356
 357    return bo->map;
 358 }
 359
 360 void lima_bo_unmap(struct lima_bo *bo)
 361 {
 362    if (bo->map) {
 363       os_munmap(bo->map, bo->size);
 364       bo->map = NULL;
 365    }
 366 }
 367
 368 bool lima_bo_export(struct lima_bo *bo, struct winsys_handle *handle)
 369 {
 370    struct lima_screen *screen = bo->screen;
 371
 372    /* Don't cache exported BOs */
 373    bo->cacheable = false;
 374
 375    switch (handle->type) {
 376    case WINSYS_HANDLE_TYPE_SHARED:
 377       if (!bo->flink_name) {
 378          struct drm_gem_flink flink = {
 379             .handle = bo->handle,
 380             .name = 0,
 381          };
 382          if (drmIoctl(screen->fd, DRM_IOCTL_GEM_FLINK, &flink))
 383             return false;
 384
 385          bo->flink_name = flink.name;
 386
 387          mtx_lock(&screen->bo_table_lock);
 388          util_hash_table_set(screen->bo_flink_names,
 389                              (void *)(uintptr_t)bo->flink_name, bo);
 390          mtx_unlock(&screen->bo_table_lock);
 391       }
 392       handle->handle = bo->flink_name;
 393       return true;
 394
 395    case WINSYS_HANDLE_TYPE_KMS:
 396       mtx_lock(&screen->bo_table_lock);
 397       util_hash_table_set(screen->bo_handles,
 398                           (void *)(uintptr_t)bo->handle, bo);
 399       mtx_unlock(&screen->bo_table_lock);
 400
 401       handle->handle = bo->handle;
 402       return true;
 403
 404    case WINSYS_HANDLE_TYPE_FD:
 405       if (drmPrimeHandleToFD(screen->fd, bo->handle, DRM_CLOEXEC,
 406                              (int*)&handle->handle))
 407          return false;
 408
 409       mtx_lock(&screen->bo_table_lock);
 410       util_hash_table_set(screen->bo_handles,
 411                           (void *)(uintptr_t)bo->handle, bo);
 412       mtx_unlock(&screen->bo_table_lock);
 413       return true;
 414
 415    default:
 416       return false;
 417    }
 418 }
 419
 420 struct lima_bo *lima_bo_import(struct lima_screen *screen,
 421                                struct winsys_handle *handle)
 422 {
 423    struct lima_bo *bo = NULL;
 424    struct drm_gem_open req = {0};
 425    uint32_t dma_buf_size = 0;
 426    unsigned h = handle->handle;
 427
 428    mtx_lock(&screen->bo_table_lock);
 429
 430    /* Convert a DMA buf handle to a KMS handle now. */
 431    if (handle->type == WINSYS_HANDLE_TYPE_FD) {
 432       uint32_t prime_handle;
 433       off_t size;
 434
 435       /* Get a KMS handle. */
 436       if (drmPrimeFDToHandle(screen->fd, h, &prime_handle)) {
 437          mtx_unlock(&screen->bo_table_lock);
 438          return NULL;
 439       }
 440
 441       /* Query the buffer size. */
 442       size = lseek(h, 0, SEEK_END);
 443       if (size == (off_t)-1) {
 444          mtx_unlock(&screen->bo_table_lock);
 445          lima_close_kms_handle(screen, prime_handle);
 446          return NULL;
 447       }
 448       lseek(h, 0, SEEK_SET);
 449
 450       dma_buf_size = size;
 451       h = prime_handle;
 452    }
 453
 454    switch (handle->type) {
 455    case WINSYS_HANDLE_TYPE_SHARED:
 456       bo = util_hash_table_get(screen->bo_flink_names,
 457                                (void *)(uintptr_t)h);
 458       break;
 459    case WINSYS_HANDLE_TYPE_KMS:
 460    case WINSYS_HANDLE_TYPE_FD:
 461       bo = util_hash_table_get(screen->bo_handles,
 462                                (void *)(uintptr_t)h);
 463       break;
 464    default:
 465       mtx_unlock(&screen->bo_table_lock);
 466       return NULL;
 467    }
 468
 469    if (bo) {
 470       p_atomic_inc(&bo->refcnt);
 471       /* Don't cache imported BOs */
 472       bo->cacheable = false;
 473       mtx_unlock(&screen->bo_table_lock);
 474       return bo;
 475    }
 476
 477    if (!(bo = calloc(1, sizeof(*bo)))) {
 478       mtx_unlock(&screen->bo_table_lock);
 479       if (handle->type == WINSYS_HANDLE_TYPE_FD)
 480          lima_close_kms_handle(screen, h);
 481       return NULL;
 482    }
 483
 484    /* Don't cache imported BOs */
 485    bo->cacheable = false;
 486    list_inithead(&bo->time_list);
 487    list_inithead(&bo->size_list);
 488    bo->screen = screen;
 489    p_atomic_set(&bo->refcnt, 1);
 490
 491    switch (handle->type) {
 492    case WINSYS_HANDLE_TYPE_SHARED:
 493       req.name = h;
 494       if (drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &req)) {
 495          mtx_unlock(&screen->bo_table_lock);
 496          free(bo);
 497          return NULL;
 498       }
 499       bo->handle = req.handle;
 500       bo->flink_name = h;
 501       bo->size = req.size;
 502       break;
 503    case WINSYS_HANDLE_TYPE_FD:
 504       bo->handle = h;
 505       bo->size = dma_buf_size;
 506       break;
 507    default:
 508       /* not possible */
 509       assert(0);
 510    }
 511
 512    if (lima_bo_get_info(bo)) {
 513       if (handle->type == WINSYS_HANDLE_TYPE_SHARED)
 514          util_hash_table_set(screen->bo_flink_names,
 515                           (void *)(uintptr_t)bo->flink_name, bo);
 516       util_hash_table_set(screen->bo_handles,
 517                           (void*)(uintptr_t)bo->handle, bo);
 518    }
 519    else {
 520       lima_close_kms_handle(screen, bo->handle);
 521       free(bo);
 522       bo = NULL;
 523    }
 524
 525    mtx_unlock(&screen->bo_table_lock);
 526
 527    return bo;
 528 }
 529
 530 bool lima_bo_wait(struct lima_bo *bo, uint32_t op, uint64_t timeout_ns)
 531 {
 532    int64_t abs_timeout;
 533
 534    if (timeout_ns == 0)
 535       abs_timeout = 0;
 536    else
 537       abs_timeout = os_time_get_absolute_timeout(timeout_ns);
 538
 539    if (abs_timeout == OS_TIMEOUT_INFINITE)
 540       abs_timeout = INT64_MAX;
 541
 542    struct drm_lima_gem_wait req = {
 543       .handle = bo->handle,
 544       .op = op,
 545       .timeout_ns = abs_timeout,
 546    };
 547
 548    return drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_WAIT, &req) == 0;
 549 }