src/gallium/drivers/lima/lima_bo.c

   1 /*
   2  * Copyright (C) 2017-2019 Lima Project
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <stdlib.h>
  25 #include <sys/types.h>
  26 #include <unistd.h>
  27 #include <fcntl.h>
  28
  29 #include "xf86drm.h"
  30 #include "drm-uapi/lima_drm.h"
  31
  32 #include "util/u_hash_table.h"
  33 #include "util/u_math.h"
  34 #include "util/os_time.h"
  35 #include "os/os_mman.h"
  36
  37 #include "state_tracker/drm_driver.h"
  38
  39 #include "lima_screen.h"
  40 #include "lima_bo.h"
  41 #include "lima_util.h"
  42
  43 bool lima_bo_table_init(struct lima_screen *screen)
  44 {
  45    screen->bo_handles = util_hash_table_create_ptr_keys();
  46    if (!screen->bo_handles)
  47       return false;
  48
  49    screen->bo_flink_names = util_hash_table_create_ptr_keys();
  50    if (!screen->bo_flink_names)
  51       goto err_out0;
  52
  53    mtx_init(&screen->bo_table_lock, mtx_plain);
  54    return true;
  55
  56 err_out0:
  57    _mesa_hash_table_destroy(screen->bo_handles, NULL);
  58    return false;
  59 }
  60
  61 bool lima_bo_cache_init(struct lima_screen *screen)
  62 {
  63    mtx_init(&screen->bo_cache_lock, mtx_plain);
  64    list_inithead(&screen->bo_cache_time);
  65    for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++)
  66       list_inithead(&screen->bo_cache_buckets[i]);
  67
  68    return true;
  69 }
  70
  71 void lima_bo_table_fini(struct lima_screen *screen)
  72 {
  73    mtx_destroy(&screen->bo_table_lock);
  74    _mesa_hash_table_destroy(screen->bo_handles, NULL);
  75    _mesa_hash_table_destroy(screen->bo_flink_names, NULL);
  76 }
  77
  78 static void
  79 lima_bo_cache_remove(struct lima_bo *bo)
  80 {
  81    list_del(&bo->size_list);
  82    list_del(&bo->time_list);
  83 }
  84
  85 static void lima_close_kms_handle(struct lima_screen *screen, uint32_t handle)
  86 {
  87    struct drm_gem_close args = {
  88       .handle = handle,
  89    };
  90
  91    drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &args);
  92 }
  93
  94 static void
  95 lima_bo_free(struct lima_bo *bo)
  96 {
  97    struct lima_screen *screen = bo->screen;
  98
  99    if (lima_debug & LIMA_DEBUG_BO_CACHE)
 100       fprintf(stderr, "%s: %p (size=%d)\n", __func__,
 101               bo, bo->size);
 102
 103    mtx_lock(&screen->bo_table_lock);
 104    _mesa_hash_table_remove_key(screen->bo_handles,
 105                           (void *)(uintptr_t)bo->handle);
 106    if (bo->flink_name)
 107       _mesa_hash_table_remove_key(screen->bo_flink_names,
 108                              (void *)(uintptr_t)bo->flink_name);
 109    mtx_unlock(&screen->bo_table_lock);
 110
 111    if (bo->map)
 112       lima_bo_unmap(bo);
 113
 114    lima_close_kms_handle(screen, bo->handle);
 115    free(bo);
 116 }
 117
 118 void lima_bo_cache_fini(struct lima_screen *screen)
 119 {
 120    mtx_destroy(&screen->bo_cache_lock);
 121
 122    list_for_each_entry_safe(struct lima_bo, entry,
 123                             &screen->bo_cache_time, time_list) {
 124       lima_bo_cache_remove(entry);
 125       lima_bo_free(entry);
 126    }
 127 }
 128
 129 static bool lima_bo_get_info(struct lima_bo *bo)
 130 {
 131    struct drm_lima_gem_info req = {
 132       .handle = bo->handle,
 133    };
 134
 135    if(drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_INFO, &req))
 136       return false;
 137
 138    bo->offset = req.offset;
 139    bo->va = req.va;
 140    return true;
 141 }
 142
 143 static unsigned
 144 lima_bucket_index(unsigned size)
 145 {
 146    /* Round down to POT to compute a bucket index */
 147
 148    unsigned bucket_index = util_logbase2(size);
 149
 150    /* Clamp the bucket index; all huge allocations will be
 151     * sorted into the largest bucket */
 152    bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET,
 153                         MAX_BO_CACHE_BUCKET);
 154
 155    /* Reindex from 0 */
 156    return (bucket_index - MIN_BO_CACHE_BUCKET);
 157 }
 158
 159 static struct list_head *
 160 lima_bo_cache_get_bucket(struct lima_screen *screen, unsigned size)
 161 {
 162    return &screen->bo_cache_buckets[lima_bucket_index(size)];
 163 }
 164
 165 static void
 166 lima_bo_cache_free_stale_bos(struct lima_screen *screen, time_t time)
 167 {
 168    unsigned cnt = 0;
 169    list_for_each_entry_safe(struct lima_bo, entry,
 170                             &screen->bo_cache_time, time_list) {
 171       /* Free BOs that are sitting idle for longer than 5 seconds */
 172       if (time - entry->free_time > 6) {
 173          lima_bo_cache_remove(entry);
 174          lima_bo_free(entry);
 175          cnt++;
 176       } else
 177          break;
 178    }
 179    if ((lima_debug & LIMA_DEBUG_BO_CACHE) && cnt)
 180       fprintf(stderr, "%s: freed %d stale BOs\n", __func__, cnt);
 181 }
 182
 183 static void
 184 lima_bo_cache_print_stats(struct lima_screen *screen)
 185 {
 186    fprintf(stderr, "===============\n");
 187    fprintf(stderr, "BO cache stats:\n");
 188    unsigned total_size = 0;
 189    for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++) {
 190       struct list_head *bucket = &screen->bo_cache_buckets[i];
 191       unsigned bucket_size = 0;
 192       list_for_each_entry(struct lima_bo, entry, bucket, size_list) {
 193          bucket_size += entry->size;
 194          total_size += entry->size;
 195       }
 196       fprintf(stderr, "Bucket #%d, BOs: %d, size: %u\n", i,
 197               list_length(bucket),
 198               bucket_size);
 199    }
 200    fprintf(stderr, "Total size: %u\n", total_size);
 201 }
 202
 203 static bool
 204 lima_bo_cache_put(struct lima_bo *bo)
 205 {
 206    if (!bo->cacheable)
 207       return false;
 208
 209    struct lima_screen *screen = bo->screen;
 210
 211    mtx_lock(&screen->bo_cache_lock);
 212    struct list_head *bucket = lima_bo_cache_get_bucket(screen, bo->size);
 213
 214    if (!bucket) {
 215       mtx_unlock(&screen->bo_cache_lock);
 216       return false;
 217    }
 218
 219    struct timespec time;
 220    clock_gettime(CLOCK_MONOTONIC, &time);
 221    bo->free_time = time.tv_sec;
 222    list_addtail(&bo->size_list, bucket);
 223    list_addtail(&bo->time_list, &screen->bo_cache_time);
 224    lima_bo_cache_free_stale_bos(screen, time.tv_sec);
 225    if (lima_debug & LIMA_DEBUG_BO_CACHE) {
 226       fprintf(stderr, "%s: put BO: %p (size=%d)\n", __func__, bo, bo->size);
 227       lima_bo_cache_print_stats(screen);
 228    }
 229    mtx_unlock(&screen->bo_cache_lock);
 230
 231    return true;
 232 }
 233
 234 static struct lima_bo *
 235 lima_bo_cache_get(struct lima_screen *screen, uint32_t size, uint32_t flags)
 236 {
 237    /* we won't cache heap buffer */
 238    if (flags & LIMA_BO_FLAG_HEAP)
 239       return NULL;
 240
 241    struct lima_bo *bo = NULL;
 242    mtx_lock(&screen->bo_cache_lock);
 243    struct list_head *bucket = lima_bo_cache_get_bucket(screen, size);
 244
 245    if (!bucket) {
 246       mtx_unlock(&screen->bo_cache_lock);
 247       return false;
 248    }
 249
 250    list_for_each_entry_safe(struct lima_bo, entry, bucket, size_list) {
 251       if (entry->size >= size) {
 252          /* Check if BO is idle. If it's not it's better to allocate new one */
 253          if (!lima_bo_wait(entry, LIMA_GEM_WAIT_WRITE, 0)) {
 254             if (lima_debug & LIMA_DEBUG_BO_CACHE) {
 255                fprintf(stderr, "%s: found BO %p but it's busy\n", __func__,
 256                        entry);
 257             }
 258             break;
 259          }
 260
 261          lima_bo_cache_remove(entry);
 262          p_atomic_set(&entry->refcnt, 1);
 263          entry->flags = flags;
 264          bo = entry;
 265          if (lima_debug & LIMA_DEBUG_BO_CACHE) {
 266             fprintf(stderr, "%s: got BO: %p (size=%d), requested size %d\n",
 267                     __func__, bo, bo->size, size);
 268             lima_bo_cache_print_stats(screen);
 269          }
 270          break;
 271       }
 272    }
 273
 274    mtx_unlock(&screen->bo_cache_lock);
 275
 276    return bo;
 277 }
 278
 279 struct lima_bo *lima_bo_create(struct lima_screen *screen,
 280                                uint32_t size, uint32_t flags)
 281 {
 282    struct lima_bo *bo;
 283
 284    size = align(size, LIMA_PAGE_SIZE);
 285
 286    /* Try to get bo from cache first */
 287    bo = lima_bo_cache_get(screen, size, flags);
 288    if (bo)
 289       return bo;
 290
 291    struct drm_lima_gem_create req = {
 292       .size = size,
 293       .flags = flags,
 294    };
 295
 296    if (!(bo = calloc(1, sizeof(*bo))))
 297       return NULL;
 298
 299    list_inithead(&bo->time_list);
 300    list_inithead(&bo->size_list);
 301
 302    if (drmIoctl(screen->fd, DRM_IOCTL_LIMA_GEM_CREATE, &req))
 303       goto err_out0;
 304
 305    bo->screen = screen;
 306    bo->size = req.size;
 307    bo->flags = req.flags;
 308    bo->handle = req.handle;
 309    bo->cacheable = !(lima_debug & LIMA_DEBUG_NO_BO_CACHE ||
 310                      flags & LIMA_BO_FLAG_HEAP);
 311    p_atomic_set(&bo->refcnt, 1);
 312
 313    if (!lima_bo_get_info(bo))
 314       goto err_out1;
 315
 316    if (lima_debug & LIMA_DEBUG_BO_CACHE)
 317       fprintf(stderr, "%s: %p (size=%d)\n", __func__,
 318               bo, bo->size);
 319
 320    return bo;
 321
 322 err_out1:
 323    lima_close_kms_handle(screen, bo->handle);
 324 err_out0:
 325    free(bo);
 326    return NULL;
 327 }
 328
 329 void lima_bo_unreference(struct lima_bo *bo)
 330 {
 331    if (!p_atomic_dec_zero(&bo->refcnt))
 332       return;
 333
 334    /* Try to put it into cache */
 335    if (lima_bo_cache_put(bo))
 336       return;
 337
 338    lima_bo_free(bo);
 339 }
 340
 341 void *lima_bo_map(struct lima_bo *bo)
 342 {
 343    if (!bo->map) {
 344       bo->map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE,
 345                         MAP_SHARED, bo->screen->fd, bo->offset);
 346       if (bo->map == MAP_FAILED)
 347           bo->map = NULL;
 348    }
 349
 350    return bo->map;
 351 }
 352
 353 void lima_bo_unmap(struct lima_bo *bo)
 354 {
 355    if (bo->map) {
 356       os_munmap(bo->map, bo->size);
 357       bo->map = NULL;
 358    }
 359 }
 360
 361 bool lima_bo_export(struct lima_bo *bo, struct winsys_handle *handle)
 362 {
 363    struct lima_screen *screen = bo->screen;
 364
 365    /* Don't cache exported BOs */
 366    bo->cacheable = false;
 367
 368    switch (handle->type) {
 369    case WINSYS_HANDLE_TYPE_SHARED:
 370       if (!bo->flink_name) {
 371          struct drm_gem_flink flink = {
 372             .handle = bo->handle,
 373             .name = 0,
 374          };
 375          if (drmIoctl(screen->fd, DRM_IOCTL_GEM_FLINK, &flink))
 376             return false;
 377
 378          bo->flink_name = flink.name;
 379
 380          mtx_lock(&screen->bo_table_lock);
 381          _mesa_hash_table_insert(screen->bo_flink_names,
 382                              (void *)(uintptr_t)bo->flink_name, bo);
 383          mtx_unlock(&screen->bo_table_lock);
 384       }
 385       handle->handle = bo->flink_name;
 386       return true;
 387
 388    case WINSYS_HANDLE_TYPE_KMS:
 389       mtx_lock(&screen->bo_table_lock);
 390       _mesa_hash_table_insert(screen->bo_handles,
 391                           (void *)(uintptr_t)bo->handle, bo);
 392       mtx_unlock(&screen->bo_table_lock);
 393
 394       handle->handle = bo->handle;
 395       return true;
 396
 397    case WINSYS_HANDLE_TYPE_FD:
 398       if (drmPrimeHandleToFD(screen->fd, bo->handle, DRM_CLOEXEC,
 399                              (int*)&handle->handle))
 400          return false;
 401
 402       mtx_lock(&screen->bo_table_lock);
 403       _mesa_hash_table_insert(screen->bo_handles,
 404                           (void *)(uintptr_t)bo->handle, bo);
 405       mtx_unlock(&screen->bo_table_lock);
 406       return true;
 407
 408    default:
 409       return false;
 410    }
 411 }
 412
 413 struct lima_bo *lima_bo_import(struct lima_screen *screen,
 414                                struct winsys_handle *handle)
 415 {
 416    struct lima_bo *bo = NULL;
 417    struct drm_gem_open req = {0};
 418    uint32_t dma_buf_size = 0;
 419    unsigned h = handle->handle;
 420
 421    mtx_lock(&screen->bo_table_lock);
 422
 423    /* Convert a DMA buf handle to a KMS handle now. */
 424    if (handle->type == WINSYS_HANDLE_TYPE_FD) {
 425       uint32_t prime_handle;
 426       off_t size;
 427
 428       /* Get a KMS handle. */
 429       if (drmPrimeFDToHandle(screen->fd, h, &prime_handle)) {
 430          mtx_unlock(&screen->bo_table_lock);
 431          return NULL;
 432       }
 433
 434       /* Query the buffer size. */
 435       size = lseek(h, 0, SEEK_END);
 436       if (size == (off_t)-1) {
 437          mtx_unlock(&screen->bo_table_lock);
 438          lima_close_kms_handle(screen, prime_handle);
 439          return NULL;
 440       }
 441       lseek(h, 0, SEEK_SET);
 442
 443       dma_buf_size = size;
 444       h = prime_handle;
 445    }
 446
 447    switch (handle->type) {
 448    case WINSYS_HANDLE_TYPE_SHARED:
 449       bo = util_hash_table_get(screen->bo_flink_names,
 450                                (void *)(uintptr_t)h);
 451       break;
 452    case WINSYS_HANDLE_TYPE_KMS:
 453    case WINSYS_HANDLE_TYPE_FD:
 454       bo = util_hash_table_get(screen->bo_handles,
 455                                (void *)(uintptr_t)h);
 456       break;
 457    default:
 458       mtx_unlock(&screen->bo_table_lock);
 459       return NULL;
 460    }
 461
 462    if (bo) {
 463       p_atomic_inc(&bo->refcnt);
 464       /* Don't cache imported BOs */
 465       bo->cacheable = false;
 466       mtx_unlock(&screen->bo_table_lock);
 467       return bo;
 468    }
 469
 470    if (!(bo = calloc(1, sizeof(*bo)))) {
 471       mtx_unlock(&screen->bo_table_lock);
 472       if (handle->type == WINSYS_HANDLE_TYPE_FD)
 473          lima_close_kms_handle(screen, h);
 474       return NULL;
 475    }
 476
 477    /* Don't cache imported BOs */
 478    bo->cacheable = false;
 479    list_inithead(&bo->time_list);
 480    list_inithead(&bo->size_list);
 481    bo->screen = screen;
 482    p_atomic_set(&bo->refcnt, 1);
 483
 484    switch (handle->type) {
 485    case WINSYS_HANDLE_TYPE_SHARED:
 486       req.name = h;
 487       if (drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &req)) {
 488          mtx_unlock(&screen->bo_table_lock);
 489          free(bo);
 490          return NULL;
 491       }
 492       bo->handle = req.handle;
 493       bo->flink_name = h;
 494       bo->size = req.size;
 495       break;
 496    case WINSYS_HANDLE_TYPE_FD:
 497       bo->handle = h;
 498       bo->size = dma_buf_size;
 499       break;
 500    default:
 501       /* not possible */
 502       assert(0);
 503    }
 504
 505    if (lima_bo_get_info(bo)) {
 506       if (handle->type == WINSYS_HANDLE_TYPE_SHARED)
 507          _mesa_hash_table_insert(screen->bo_flink_names,
 508                           (void *)(uintptr_t)bo->flink_name, bo);
 509       _mesa_hash_table_insert(screen->bo_handles,
 510                           (void*)(uintptr_t)bo->handle, bo);
 511    }
 512    else {
 513       lima_close_kms_handle(screen, bo->handle);
 514       free(bo);
 515       bo = NULL;
 516    }
 517
 518    mtx_unlock(&screen->bo_table_lock);
 519
 520    return bo;
 521 }
 522
 523 bool lima_bo_wait(struct lima_bo *bo, uint32_t op, uint64_t timeout_ns)
 524 {
 525    int64_t abs_timeout;
 526
 527    if (timeout_ns == 0)
 528       abs_timeout = 0;
 529    else
 530       abs_timeout = os_time_get_absolute_timeout(timeout_ns);
 531
 532    if (abs_timeout == OS_TIMEOUT_INFINITE)
 533       abs_timeout = INT64_MAX;
 534
 535    struct drm_lima_gem_wait req = {
 536       .handle = bo->handle,
 537       .op = op,
 538       .timeout_ns = abs_timeout,
 539    };
 540
 541    return drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_WAIT, &req) == 0;
 542 }