src/gallium/drivers/lima/lima_bo.c

   1 /*
   2  * Copyright (C) 2017-2019 Lima Project
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <stdlib.h>
  25 #include <sys/types.h>
  26 #include <unistd.h>
  27 #include <fcntl.h>
  28
  29 #include "xf86drm.h"
  30 #include "drm-uapi/lima_drm.h"
  31
  32 #include "util/u_hash_table.h"
  33 #include "util/u_math.h"
  34 #include "util/os_time.h"
  35 #include "os/os_mman.h"
  36
  37 #include "state_tracker/drm_driver.h"
  38
  39 #include "lima_screen.h"
  40 #include "lima_bo.h"
  41 #include "lima_util.h"
  42
  43 #define PTR_TO_UINT(x) ((unsigned)((intptr_t)(x)))
  44
  45 static unsigned handle_hash(void *key)
  46 {
  47     return PTR_TO_UINT(key);
  48 }
  49
  50 static int handle_compare(void *key1, void *key2)
  51 {
  52     return PTR_TO_UINT(key1) != PTR_TO_UINT(key2);
  53 }
  54
  55 bool lima_bo_table_init(struct lima_screen *screen)
  56 {
  57    screen->bo_handles = util_hash_table_create(handle_hash, handle_compare);
  58    if (!screen->bo_handles)
  59       return false;
  60
  61    screen->bo_flink_names = util_hash_table_create(handle_hash, handle_compare);
  62    if (!screen->bo_flink_names)
  63       goto err_out0;
  64
  65    mtx_init(&screen->bo_table_lock, mtx_plain);
  66    return true;
  67
  68 err_out0:
  69    util_hash_table_destroy(screen->bo_handles);
  70    return false;
  71 }
  72
  73 bool lima_bo_cache_init(struct lima_screen *screen)
  74 {
  75    mtx_init(&screen->bo_cache_lock, mtx_plain);
  76    list_inithead(&screen->bo_cache_time);
  77    for (int i = 0; i < NR_BO_CACHE_BUCKETS; i++)
  78       list_inithead(&screen->bo_cache_buckets[i]);
  79
  80    return true;
  81 }
  82
  83 void lima_bo_table_fini(struct lima_screen *screen)
  84 {
  85    mtx_destroy(&screen->bo_table_lock);
  86    util_hash_table_destroy(screen->bo_handles);
  87    util_hash_table_destroy(screen->bo_flink_names);
  88 }
  89
  90 static void
  91 lima_bo_cache_remove(struct lima_bo *bo)
  92 {
  93    list_del(&bo->size_list);
  94    list_del(&bo->time_list);
  95 }
  96
  97 static void lima_close_kms_handle(struct lima_screen *screen, uint32_t handle)
  98 {
  99    struct drm_gem_close args = {
 100       .handle = handle,
 101    };
 102
 103    drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &args);
 104 }
 105
 106 static void
 107 lima_bo_free(struct lima_bo *bo)
 108 {
 109    struct lima_screen *screen = bo->screen;
 110    mtx_lock(&screen->bo_table_lock);
 111    util_hash_table_remove(screen->bo_handles,
 112                           (void *)(uintptr_t)bo->handle);
 113    if (bo->flink_name)
 114       util_hash_table_remove(screen->bo_flink_names,
 115                              (void *)(uintptr_t)bo->flink_name);
 116    mtx_unlock(&screen->bo_table_lock);
 117
 118    if (bo->map)
 119       lima_bo_unmap(bo);
 120
 121    lima_close_kms_handle(screen, bo->handle);
 122    free(bo);
 123 }
 124
 125 void lima_bo_cache_fini(struct lima_screen *screen)
 126 {
 127    mtx_destroy(&screen->bo_cache_lock);
 128
 129    list_for_each_entry_safe(struct lima_bo, entry,
 130                             &screen->bo_cache_time, time_list) {
 131       lima_bo_cache_remove(entry);
 132       lima_bo_free(entry);
 133    }
 134 }
 135
 136 static bool lima_bo_get_info(struct lima_bo *bo)
 137 {
 138    struct drm_lima_gem_info req = {
 139       .handle = bo->handle,
 140    };
 141
 142    if(drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_INFO, &req))
 143       return false;
 144
 145    bo->offset = req.offset;
 146    bo->va = req.va;
 147    return true;
 148 }
 149
 150 static unsigned
 151 lima_bucket_index(unsigned size)
 152 {
 153    /* Round down to POT to compute a bucket index */
 154
 155    unsigned bucket_index = util_logbase2(size);
 156
 157    /* Clamp the bucket index; all huge allocations will be
 158     * sorted into the largest bucket */
 159    bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET,
 160                         MAX_BO_CACHE_BUCKET);
 161
 162    /* Reindex from 0 */
 163    return (bucket_index - MIN_BO_CACHE_BUCKET);
 164 }
 165
 166 static struct list_head *
 167 lima_bo_cache_get_bucket(struct lima_screen *screen, unsigned size)
 168 {
 169    return &screen->bo_cache_buckets[lima_bucket_index(size)];
 170 }
 171
 172 static void
 173 lima_bo_cache_free_stale_bos(struct lima_screen *screen, time_t time)
 174 {
 175    list_for_each_entry_safe(struct lima_bo, entry,
 176                             &screen->bo_cache_time, time_list) {
 177       /* Free BOs that are sitting idle for longer than 5 seconds */
 178       if (time - entry->free_time > 6) {
 179          lima_bo_cache_remove(entry);
 180          lima_bo_free(entry);
 181       } else
 182          break;
 183    }
 184 }
 185
 186 static bool
 187 lima_bo_cache_put(struct lima_bo *bo)
 188 {
 189    if (!bo->cacheable)
 190       return false;
 191
 192    struct lima_screen *screen = bo->screen;
 193
 194    mtx_lock(&screen->bo_cache_lock);
 195    struct list_head *bucket = lima_bo_cache_get_bucket(screen, bo->size);
 196
 197    if (!bucket) {
 198       mtx_unlock(&screen->bo_cache_lock);
 199       return false;
 200    }
 201
 202    struct timespec time;
 203    clock_gettime(CLOCK_MONOTONIC, &time);
 204    bo->free_time = time.tv_sec;
 205    list_addtail(&bo->size_list, bucket);
 206    list_addtail(&bo->time_list, &screen->bo_cache_time);
 207    lima_bo_cache_free_stale_bos(screen, time.tv_sec);
 208    mtx_unlock(&screen->bo_cache_lock);
 209
 210    return true;
 211 }
 212
 213 static struct lima_bo *
 214 lima_bo_cache_get(struct lima_screen *screen, uint32_t size, uint32_t flags)
 215 {
 216    struct lima_bo *bo = NULL;
 217    mtx_lock(&screen->bo_cache_lock);
 218    struct list_head *bucket = lima_bo_cache_get_bucket(screen, size);
 219
 220    if (!bucket) {
 221       mtx_unlock(&screen->bo_cache_lock);
 222       return false;
 223    }
 224
 225    list_for_each_entry_safe(struct lima_bo, entry, bucket, size_list) {
 226       if (entry->size >= size &&
 227           entry->flags == flags) {
 228          /* Check if BO is idle. If it's not it's better to allocate new one */
 229          if (!lima_bo_wait(entry, LIMA_GEM_WAIT_WRITE, 0))
 230             break;
 231
 232          lima_bo_cache_remove(entry);
 233          p_atomic_set(&entry->refcnt, 1);
 234          bo = entry;
 235          break;
 236       }
 237    }
 238
 239    mtx_unlock(&screen->bo_cache_lock);
 240
 241    return bo;
 242 }
 243
 244 struct lima_bo *lima_bo_create(struct lima_screen *screen,
 245                                uint32_t size, uint32_t flags)
 246 {
 247    struct lima_bo *bo;
 248
 249    /* Try to get bo from cache first */
 250    bo = lima_bo_cache_get(screen, size, flags);
 251    if (bo)
 252       return bo;
 253
 254    size = align(size, LIMA_PAGE_SIZE);
 255
 256    struct drm_lima_gem_create req = {
 257       .size = size,
 258       .flags = flags,
 259    };
 260
 261    if (!(bo = calloc(1, sizeof(*bo))))
 262       return NULL;
 263
 264    list_inithead(&bo->time_list);
 265    list_inithead(&bo->size_list);
 266
 267    if (drmIoctl(screen->fd, DRM_IOCTL_LIMA_GEM_CREATE, &req))
 268       goto err_out0;
 269
 270    bo->screen = screen;
 271    bo->size = req.size;
 272    bo->flags = req.flags;
 273    bo->handle = req.handle;
 274    bo->cacheable = !(lima_debug & LIMA_DEBUG_NO_BO_CACHE);
 275    p_atomic_set(&bo->refcnt, 1);
 276
 277    if (!lima_bo_get_info(bo))
 278       goto err_out1;
 279
 280    return bo;
 281
 282 err_out1:
 283    lima_close_kms_handle(screen, bo->handle);
 284 err_out0:
 285    free(bo);
 286    return NULL;
 287 }
 288
 289 void lima_bo_unreference(struct lima_bo *bo)
 290 {
 291    if (!p_atomic_dec_zero(&bo->refcnt))
 292       return;
 293
 294    /* Try to put it into cache */
 295    if (lima_bo_cache_put(bo))
 296       return;
 297
 298    lima_bo_free(bo);
 299 }
 300
 301 void *lima_bo_map(struct lima_bo *bo)
 302 {
 303    if (!bo->map) {
 304       bo->map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE,
 305                         MAP_SHARED, bo->screen->fd, bo->offset);
 306       if (bo->map == MAP_FAILED)
 307           bo->map = NULL;
 308    }
 309
 310    return bo->map;
 311 }
 312
 313 void lima_bo_unmap(struct lima_bo *bo)
 314 {
 315    if (bo->map) {
 316       os_munmap(bo->map, bo->size);
 317       bo->map = NULL;
 318    }
 319 }
 320
 321 bool lima_bo_export(struct lima_bo *bo, struct winsys_handle *handle)
 322 {
 323    struct lima_screen *screen = bo->screen;
 324
 325    /* Don't cache exported BOs */
 326    bo->cacheable = false;
 327
 328    switch (handle->type) {
 329    case WINSYS_HANDLE_TYPE_SHARED:
 330       if (!bo->flink_name) {
 331          struct drm_gem_flink flink = {
 332             .handle = bo->handle,
 333             .name = 0,
 334          };
 335          if (drmIoctl(screen->fd, DRM_IOCTL_GEM_FLINK, &flink))
 336             return false;
 337
 338          bo->flink_name = flink.name;
 339
 340          mtx_lock(&screen->bo_table_lock);
 341          util_hash_table_set(screen->bo_flink_names,
 342                              (void *)(uintptr_t)bo->flink_name, bo);
 343          mtx_unlock(&screen->bo_table_lock);
 344       }
 345       handle->handle = bo->flink_name;
 346       return true;
 347
 348    case WINSYS_HANDLE_TYPE_KMS:
 349       mtx_lock(&screen->bo_table_lock);
 350       util_hash_table_set(screen->bo_handles,
 351                           (void *)(uintptr_t)bo->handle, bo);
 352       mtx_unlock(&screen->bo_table_lock);
 353
 354       handle->handle = bo->handle;
 355       return true;
 356
 357    case WINSYS_HANDLE_TYPE_FD:
 358       if (drmPrimeHandleToFD(screen->fd, bo->handle, DRM_CLOEXEC,
 359                              (int*)&handle->handle))
 360          return false;
 361
 362       mtx_lock(&screen->bo_table_lock);
 363       util_hash_table_set(screen->bo_handles,
 364                           (void *)(uintptr_t)bo->handle, bo);
 365       mtx_unlock(&screen->bo_table_lock);
 366       return true;
 367
 368    default:
 369       return false;
 370    }
 371 }
 372
 373 struct lima_bo *lima_bo_import(struct lima_screen *screen,
 374                                struct winsys_handle *handle)
 375 {
 376    struct lima_bo *bo = NULL;
 377    struct drm_gem_open req = {0};
 378    uint32_t dma_buf_size = 0;
 379    unsigned h = handle->handle;
 380
 381    mtx_lock(&screen->bo_table_lock);
 382
 383    /* Convert a DMA buf handle to a KMS handle now. */
 384    if (handle->type == WINSYS_HANDLE_TYPE_FD) {
 385       uint32_t prime_handle;
 386       off_t size;
 387
 388       /* Get a KMS handle. */
 389       if (drmPrimeFDToHandle(screen->fd, h, &prime_handle)) {
 390          mtx_unlock(&screen->bo_table_lock);
 391          return NULL;
 392       }
 393
 394       /* Query the buffer size. */
 395       size = lseek(h, 0, SEEK_END);
 396       if (size == (off_t)-1) {
 397          mtx_unlock(&screen->bo_table_lock);
 398          lima_close_kms_handle(screen, prime_handle);
 399          return NULL;
 400       }
 401       lseek(h, 0, SEEK_SET);
 402
 403       dma_buf_size = size;
 404       h = prime_handle;
 405    }
 406
 407    switch (handle->type) {
 408    case WINSYS_HANDLE_TYPE_SHARED:
 409       bo = util_hash_table_get(screen->bo_flink_names,
 410                                (void *)(uintptr_t)h);
 411       break;
 412    case WINSYS_HANDLE_TYPE_KMS:
 413    case WINSYS_HANDLE_TYPE_FD:
 414       bo = util_hash_table_get(screen->bo_handles,
 415                                (void *)(uintptr_t)h);
 416       break;
 417    default:
 418       mtx_unlock(&screen->bo_table_lock);
 419       return NULL;
 420    }
 421
 422    if (bo) {
 423       p_atomic_inc(&bo->refcnt);
 424       /* Don't cache imported BOs */
 425       bo->cacheable = false;
 426       mtx_unlock(&screen->bo_table_lock);
 427       return bo;
 428    }
 429
 430    if (!(bo = calloc(1, sizeof(*bo)))) {
 431       mtx_unlock(&screen->bo_table_lock);
 432       if (handle->type == WINSYS_HANDLE_TYPE_FD)
 433          lima_close_kms_handle(screen, h);
 434       return NULL;
 435    }
 436
 437    /* Don't cache imported BOs */
 438    bo->cacheable = false;
 439    list_inithead(&bo->time_list);
 440    list_inithead(&bo->size_list);
 441    bo->screen = screen;
 442    p_atomic_set(&bo->refcnt, 1);
 443
 444    switch (handle->type) {
 445    case WINSYS_HANDLE_TYPE_SHARED:
 446       req.name = h;
 447       if (drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &req)) {
 448          mtx_unlock(&screen->bo_table_lock);
 449          free(bo);
 450          return NULL;
 451       }
 452       bo->handle = req.handle;
 453       bo->flink_name = h;
 454       bo->size = req.size;
 455       break;
 456    case WINSYS_HANDLE_TYPE_FD:
 457       bo->handle = h;
 458       bo->size = dma_buf_size;
 459       break;
 460    default:
 461       /* not possible */
 462       assert(0);
 463    }
 464
 465    if (lima_bo_get_info(bo)) {
 466       if (handle->type == WINSYS_HANDLE_TYPE_SHARED)
 467          util_hash_table_set(screen->bo_flink_names,
 468                           (void *)(uintptr_t)bo->flink_name, bo);
 469       util_hash_table_set(screen->bo_handles,
 470                           (void*)(uintptr_t)bo->handle, bo);
 471    }
 472    else {
 473       lima_close_kms_handle(screen, bo->handle);
 474       free(bo);
 475       bo = NULL;
 476    }
 477
 478    mtx_unlock(&screen->bo_table_lock);
 479
 480    return bo;
 481 }
 482
 483 bool lima_bo_wait(struct lima_bo *bo, uint32_t op, uint64_t timeout_ns)
 484 {
 485    int64_t abs_timeout;
 486
 487    if (timeout_ns == 0)
 488       abs_timeout = 0;
 489    else
 490       abs_timeout = os_time_get_absolute_timeout(timeout_ns);
 491
 492    if (abs_timeout == OS_TIMEOUT_INFINITE)
 493       abs_timeout = INT64_MAX;
 494
 495    struct drm_lima_gem_wait req = {
 496       .handle = bo->handle,
 497       .op = op,
 498       .timeout_ns = abs_timeout,
 499    };
 500
 501    return drmIoctl(bo->screen->fd, DRM_IOCTL_LIMA_GEM_WAIT, &req) == 0;
 502 }