src/gallium/drivers/vc4/vc4_bufmgr.c

   1 /*
   2  * Copyright © 2014-2015 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <errno.h>
  25 #include <err.h>
  26 #include <sys/mman.h>
  27 #include <fcntl.h>
  28 #include <xf86drm.h>
  29 #include <xf86drmMode.h>
  30
  31 #include "util/u_hash_table.h"
  32 #include "util/u_memory.h"
  33 #include "util/ralloc.h"
  34
  35 #include "vc4_context.h"
  36 #include "vc4_screen.h"
  37
  38 #ifdef HAVE_VALGRIND
  39 #include <valgrind.h>
  40 #include <memcheck.h>
  41 #define VG(x) x
  42 #else
  43 #define VG(x)
  44 #endif
  45
  46 static bool dump_stats = false;
  47
  48 static void
  49 vc4_bo_cache_free_all(struct vc4_bo_cache *cache);
  50
  51 static void
  52 vc4_bo_dump_stats(struct vc4_screen *screen)
  53 {
  54         struct vc4_bo_cache *cache = &screen->bo_cache;
  55
  56         fprintf(stderr, "  BOs allocated:   %d\n", screen->bo_count);
  57         fprintf(stderr, "  BOs size:        %dkb\n", screen->bo_size / 1024);
  58         fprintf(stderr, "  BOs cached:      %d\n", cache->bo_count);
  59         fprintf(stderr, "  BOs cached size: %dkb\n", cache->bo_size / 1024);
  60
  61         if (!list_empty(&cache->time_list)) {
  62                 struct vc4_bo *first = LIST_ENTRY(struct vc4_bo,
  63                                                   cache->time_list.next,
  64                                                   time_list);
  65                 struct vc4_bo *last = LIST_ENTRY(struct vc4_bo,
  66                                                   cache->time_list.prev,
  67                                                   time_list);
  68
  69                 fprintf(stderr, "  oldest cache time: %ld\n",
  70                         (long)first->free_time);
  71                 fprintf(stderr, "  newest cache time: %ld\n",
  72                         (long)last->free_time);
  73
  74                 struct timespec time;
  75                 clock_gettime(CLOCK_MONOTONIC, &time);
  76                 fprintf(stderr, "  now:               %ld\n",
  77                         time.tv_sec);
  78         }
  79 }
  80
  81 static void
  82 vc4_bo_remove_from_cache(struct vc4_bo_cache *cache, struct vc4_bo *bo)
  83 {
  84         list_del(&bo->time_list);
  85         list_del(&bo->size_list);
  86         cache->bo_count--;
  87         cache->bo_size -= bo->size;
  88 }
  89
  90 static struct vc4_bo *
  91 vc4_bo_from_cache(struct vc4_screen *screen, uint32_t size, const char *name)
  92 {
  93         struct vc4_bo_cache *cache = &screen->bo_cache;
  94         uint32_t page_index = size / 4096 - 1;
  95
  96         if (cache->size_list_size <= page_index)
  97                 return NULL;
  98
  99         struct vc4_bo *bo = NULL;
 100         pipe_mutex_lock(cache->lock);
 101         if (!list_empty(&cache->size_list[page_index])) {
 102                 bo = LIST_ENTRY(struct vc4_bo, cache->size_list[page_index].next,
 103                                 size_list);
 104
 105                 /* Check that the BO has gone idle.  If not, then we want to
 106                  * allocate something new instead, since we assume that the
 107                  * user will proceed to CPU map it and fill it with stuff.
 108                  */
 109                 if (!vc4_bo_wait(bo, 0, NULL)) {
 110                         pipe_mutex_unlock(cache->lock);
 111                         return NULL;
 112                 }
 113
 114                 pipe_reference_init(&bo->reference, 1);
 115                 vc4_bo_remove_from_cache(cache, bo);
 116
 117                 bo->name = name;
 118         }
 119         pipe_mutex_unlock(cache->lock);
 120         return bo;
 121 }
 122
 123 struct vc4_bo *
 124 vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name)
 125 {
 126         struct vc4_bo *bo;
 127         int ret;
 128
 129         size = align(size, 4096);
 130
 131         bo = vc4_bo_from_cache(screen, size, name);
 132         if (bo) {
 133                 if (dump_stats) {
 134                         fprintf(stderr, "Allocated %s %dkb from cache:\n",
 135                                 name, size / 1024);
 136                         vc4_bo_dump_stats(screen);
 137                 }
 138                 return bo;
 139         }
 140
 141         bo = CALLOC_STRUCT(vc4_bo);
 142         if (!bo)
 143                 return NULL;
 144
 145         pipe_reference_init(&bo->reference, 1);
 146         bo->screen = screen;
 147         bo->size = size;
 148         bo->name = name;
 149         bo->private = true;
 150
 151         bool cleared_and_retried = false;
 152 retry:
 153         if (!using_vc4_simulator) {
 154                 struct drm_vc4_create_bo create;
 155                 memset(&create, 0, sizeof(create));
 156
 157                 create.size = size;
 158
 159                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_CREATE_BO, &create);
 160                 bo->handle = create.handle;
 161         } else {
 162                 struct drm_mode_create_dumb create;
 163                 memset(&create, 0, sizeof(create));
 164
 165                 create.width = 128;
 166                 create.bpp = 8;
 167                 create.height = (size + 127) / 128;
 168
 169                 ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
 170                 bo->handle = create.handle;
 171                 assert(create.size >= size);
 172         }
 173         if (ret != 0) {
 174                 if (!list_empty(&screen->bo_cache.time_list) &&
 175                     !cleared_and_retried) {
 176                         cleared_and_retried = true;
 177                         vc4_bo_cache_free_all(&screen->bo_cache);
 178                         goto retry;
 179                 }
 180
 181                 free(bo);
 182                 return NULL;
 183         }
 184
 185         screen->bo_count++;
 186         screen->bo_size += bo->size;
 187         if (dump_stats) {
 188                 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
 189                 vc4_bo_dump_stats(screen);
 190         }
 191
 192         return bo;
 193 }
 194
 195 void
 196 vc4_bo_last_unreference(struct vc4_bo *bo)
 197 {
 198         struct vc4_screen *screen = bo->screen;
 199
 200         struct timespec time;
 201         clock_gettime(CLOCK_MONOTONIC, &time);
 202         pipe_mutex_lock(screen->bo_cache.lock);
 203         vc4_bo_last_unreference_locked_timed(bo, time.tv_sec);
 204         pipe_mutex_unlock(screen->bo_cache.lock);
 205 }
 206
 207 static void
 208 vc4_bo_free(struct vc4_bo *bo)
 209 {
 210         struct vc4_screen *screen = bo->screen;
 211
 212         if (bo->map) {
 213 #ifdef USE_VC4_SIMULATOR
 214                 if (bo->simulator_winsys_map) {
 215                         free(bo->map);
 216                         bo->map = bo->simulator_winsys_map;
 217                 }
 218 #endif
 219                 munmap(bo->map, bo->size);
 220                 VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
 221         }
 222
 223         struct drm_gem_close c;
 224         memset(&c, 0, sizeof(c));
 225         c.handle = bo->handle;
 226         int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
 227         if (ret != 0)
 228                 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
 229
 230         screen->bo_count--;
 231         screen->bo_size -= bo->size;
 232
 233         if (dump_stats) {
 234                 fprintf(stderr, "Freed %s%s%dkb:\n",
 235                         bo->name ? bo->name : "",
 236                         bo->name ? " " : "",
 237                         bo->size / 1024);
 238                 vc4_bo_dump_stats(screen);
 239         }
 240
 241         free(bo);
 242 }
 243
 244 static void
 245 free_stale_bos(struct vc4_screen *screen, time_t time)
 246 {
 247         struct vc4_bo_cache *cache = &screen->bo_cache;
 248         bool freed_any = false;
 249
 250         list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
 251                                  time_list) {
 252                 if (dump_stats && !freed_any) {
 253                         fprintf(stderr, "Freeing stale BOs:\n");
 254                         vc4_bo_dump_stats(screen);
 255                         freed_any = true;
 256                 }
 257
 258                 /* If it's more than a second old, free it. */
 259                 if (time - bo->free_time > 2) {
 260                         vc4_bo_remove_from_cache(cache, bo);
 261                         vc4_bo_free(bo);
 262                 } else {
 263                         break;
 264                 }
 265         }
 266
 267         if (dump_stats && freed_any) {
 268                 fprintf(stderr, "Freed stale BOs:\n");
 269                 vc4_bo_dump_stats(screen);
 270         }
 271 }
 272
 273 static void
 274 vc4_bo_cache_free_all(struct vc4_bo_cache *cache)
 275 {
 276         pipe_mutex_lock(cache->lock);
 277         list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
 278                                  time_list) {
 279                 vc4_bo_remove_from_cache(cache, bo);
 280                 vc4_bo_free(bo);
 281         }
 282         pipe_mutex_unlock(cache->lock);
 283 }
 284
 285 void
 286 vc4_bo_last_unreference_locked_timed(struct vc4_bo *bo, time_t time)
 287 {
 288         struct vc4_screen *screen = bo->screen;
 289         struct vc4_bo_cache *cache = &screen->bo_cache;
 290         uint32_t page_index = bo->size / 4096 - 1;
 291
 292         if (!bo->private) {
 293                 vc4_bo_free(bo);
 294                 return;
 295         }
 296
 297         if (cache->size_list_size <= page_index) {
 298                 struct list_head *new_list =
 299                         ralloc_array(screen, struct list_head, page_index + 1);
 300
 301                 /* Move old list contents over (since the array has moved, and
 302                  * therefore the pointers to the list heads have to change).
 303                  */
 304                 for (int i = 0; i < cache->size_list_size; i++) {
 305                         struct list_head *old_head = &cache->size_list[i];
 306                         if (list_empty(old_head))
 307                                 list_inithead(&new_list[i]);
 308                         else {
 309                                 new_list[i].next = old_head->next;
 310                                 new_list[i].prev = old_head->prev;
 311                                 new_list[i].next->prev = &new_list[i];
 312                                 new_list[i].prev->next = &new_list[i];
 313                         }
 314                 }
 315                 for (int i = cache->size_list_size; i < page_index + 1; i++)
 316                         list_inithead(&new_list[i]);
 317
 318                 cache->size_list = new_list;
 319                 cache->size_list_size = page_index + 1;
 320         }
 321
 322         bo->free_time = time;
 323         list_addtail(&bo->size_list, &cache->size_list[page_index]);
 324         list_addtail(&bo->time_list, &cache->time_list);
 325         cache->bo_count++;
 326         cache->bo_size += bo->size;
 327         if (dump_stats) {
 328                 fprintf(stderr, "Freed %s %dkb to cache:\n",
 329                         bo->name, bo->size / 1024);
 330                 vc4_bo_dump_stats(screen);
 331         }
 332         bo->name = NULL;
 333
 334         free_stale_bos(screen, time);
 335 }
 336
 337 static struct vc4_bo *
 338 vc4_bo_open_handle(struct vc4_screen *screen,
 339                    uint32_t winsys_stride,
 340                    uint32_t handle, uint32_t size)
 341 {
 342         struct vc4_bo *bo;
 343
 344         assert(size);
 345
 346         pipe_mutex_lock(screen->bo_handles_mutex);
 347
 348         bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
 349         if (bo) {
 350                 pipe_reference(NULL, &bo->reference);
 351                 goto done;
 352         }
 353
 354         bo = CALLOC_STRUCT(vc4_bo);
 355         pipe_reference_init(&bo->reference, 1);
 356         bo->screen = screen;
 357         bo->handle = handle;
 358         bo->size = size;
 359         bo->name = "winsys";
 360         bo->private = false;
 361
 362 #ifdef USE_VC4_SIMULATOR
 363         vc4_bo_map(bo);
 364         bo->simulator_winsys_map = bo->map;
 365         bo->simulator_winsys_stride = winsys_stride;
 366         bo->map = malloc(bo->size);
 367 #endif
 368
 369         util_hash_table_set(screen->bo_handles, (void *)(uintptr_t)handle, bo);
 370
 371 done:
 372         pipe_mutex_unlock(screen->bo_handles_mutex);
 373         return bo;
 374 }
 375
 376 struct vc4_bo *
 377 vc4_bo_open_name(struct vc4_screen *screen, uint32_t name,
 378                  uint32_t winsys_stride)
 379 {
 380         struct drm_gem_open o = {
 381                 .name = name
 382         };
 383         int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
 384         if (ret) {
 385                 fprintf(stderr, "Failed to open bo %d: %s\n",
 386                         name, strerror(errno));
 387                 return NULL;
 388         }
 389
 390         return vc4_bo_open_handle(screen, winsys_stride, o.handle, o.size);
 391 }
 392
 393 struct vc4_bo *
 394 vc4_bo_open_dmabuf(struct vc4_screen *screen, int fd, uint32_t winsys_stride)
 395 {
 396         uint32_t handle;
 397         int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
 398         int size;
 399         if (ret) {
 400                 fprintf(stderr, "Failed to get vc4 handle for dmabuf %d\n", fd);
 401                 return NULL;
 402         }
 403
 404         /* Determine the size of the bo we were handed. */
 405         size = lseek(fd, 0, SEEK_END);
 406         if (size == -1) {
 407                 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
 408                 return NULL;
 409         }
 410
 411         return vc4_bo_open_handle(screen, winsys_stride, handle, size);
 412 }
 413
 414 int
 415 vc4_bo_get_dmabuf(struct vc4_bo *bo)
 416 {
 417         int fd;
 418         int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
 419                                      O_CLOEXEC, &fd);
 420         if (ret != 0) {
 421                 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
 422                         bo->handle);
 423                 return -1;
 424         }
 425
 426         pipe_mutex_lock(bo->screen->bo_handles_mutex);
 427         bo->private = false;
 428         util_hash_table_set(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
 429         pipe_mutex_unlock(bo->screen->bo_handles_mutex);
 430
 431         return fd;
 432 }
 433
 434 struct vc4_bo *
 435 vc4_bo_alloc_shader(struct vc4_screen *screen, const void *data, uint32_t size)
 436 {
 437         struct vc4_bo *bo;
 438         int ret;
 439
 440         bo = CALLOC_STRUCT(vc4_bo);
 441         if (!bo)
 442                 return NULL;
 443
 444         pipe_reference_init(&bo->reference, 1);
 445         bo->screen = screen;
 446         bo->size = align(size, 4096);
 447         bo->name = "code";
 448         bo->private = false; /* Make sure it doesn't go back to the cache. */
 449
 450         if (!using_vc4_simulator) {
 451                 struct drm_vc4_create_shader_bo create = {
 452                         .size = size,
 453                         .data = (uintptr_t)data,
 454                 };
 455
 456                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_CREATE_SHADER_BO,
 457                                &create);
 458                 bo->handle = create.handle;
 459         } else {
 460                 struct drm_mode_create_dumb create;
 461                 memset(&create, 0, sizeof(create));
 462
 463                 create.width = 128;
 464                 create.bpp = 8;
 465                 create.height = (size + 127) / 128;
 466
 467                 ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
 468                 bo->handle = create.handle;
 469                 assert(create.size >= size);
 470
 471                 vc4_bo_map(bo);
 472                 memcpy(bo->map, data, size);
 473         }
 474         if (ret != 0) {
 475                 fprintf(stderr, "create shader ioctl failure\n");
 476                 abort();
 477         }
 478
 479         screen->bo_count++;
 480         screen->bo_size += bo->size;
 481         if (dump_stats) {
 482                 fprintf(stderr, "Allocated shader %dkb:\n", bo->size / 1024);
 483                 vc4_bo_dump_stats(screen);
 484         }
 485
 486         return bo;
 487 }
 488
 489 bool
 490 vc4_bo_flink(struct vc4_bo *bo, uint32_t *name)
 491 {
 492         struct drm_gem_flink flink = {
 493                 .handle = bo->handle,
 494         };
 495         int ret = drmIoctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
 496         if (ret) {
 497                 fprintf(stderr, "Failed to flink bo %d: %s\n",
 498                         bo->handle, strerror(errno));
 499                 free(bo);
 500                 return false;
 501         }
 502
 503         bo->private = false;
 504         *name = flink.name;
 505
 506         return true;
 507 }
 508
 509 static int vc4_wait_seqno_ioctl(int fd, uint64_t seqno, uint64_t timeout_ns)
 510 {
 511         if (using_vc4_simulator)
 512                 return 0;
 513
 514         struct drm_vc4_wait_seqno wait = {
 515                 .seqno = seqno,
 516                 .timeout_ns = timeout_ns,
 517         };
 518         int ret = drmIoctl(fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait);
 519         if (ret == -1)
 520                 return -errno;
 521         else
 522                 return 0;
 523
 524 }
 525
 526 bool
 527 vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns,
 528                const char *reason)
 529 {
 530         if (screen->finished_seqno >= seqno)
 531                 return true;
 532
 533         if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
 534                 if (vc4_wait_seqno_ioctl(screen->fd, seqno, 0) == -ETIME) {
 535                         fprintf(stderr, "Blocking on seqno %lld for %s\n",
 536                                 (long long)seqno, reason);
 537                 }
 538         }
 539
 540         int ret = vc4_wait_seqno_ioctl(screen->fd, seqno, timeout_ns);
 541         if (ret) {
 542                 if (ret != -ETIME) {
 543                         fprintf(stderr, "wait failed: %d\n", ret);
 544                         abort();
 545                 }
 546
 547                 return false;
 548         }
 549
 550         screen->finished_seqno = seqno;
 551         return true;
 552 }
 553
 554 static int vc4_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
 555 {
 556         if (using_vc4_simulator)
 557                 return 0;
 558
 559         struct drm_vc4_wait_bo wait = {
 560                 .handle = handle,
 561                 .timeout_ns = timeout_ns,
 562         };
 563         int ret = drmIoctl(fd, DRM_IOCTL_VC4_WAIT_BO, &wait);
 564         if (ret == -1)
 565                 return -errno;
 566         else
 567                 return 0;
 568
 569 }
 570
 571 bool
 572 vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns, const char *reason)
 573 {
 574         struct vc4_screen *screen = bo->screen;
 575
 576         if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
 577                 if (vc4_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
 578                         fprintf(stderr, "Blocking on %s BO for %s\n",
 579                                 bo->name, reason);
 580                 }
 581         }
 582
 583         int ret = vc4_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
 584         if (ret) {
 585                 if (ret != -ETIME) {
 586                         fprintf(stderr, "wait failed: %d\n", ret);
 587                         abort();
 588                 }
 589
 590                 return false;
 591         }
 592
 593         return true;
 594 }
 595
 596 void *
 597 vc4_bo_map_unsynchronized(struct vc4_bo *bo)
 598 {
 599         uint64_t offset;
 600         int ret;
 601
 602         if (bo->map)
 603                 return bo->map;
 604
 605         if (!using_vc4_simulator) {
 606                 struct drm_vc4_mmap_bo map;
 607                 memset(&map, 0, sizeof(map));
 608                 map.handle = bo->handle;
 609                 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_VC4_MMAP_BO, &map);
 610                 offset = map.offset;
 611         } else {
 612                 struct drm_mode_map_dumb map;
 613                 memset(&map, 0, sizeof(map));
 614                 map.handle = bo->handle;
 615                 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_MODE_MAP_DUMB, &map);
 616                 offset = map.offset;
 617         }
 618         if (ret != 0) {
 619                 fprintf(stderr, "map ioctl failure\n");
 620                 abort();
 621         }
 622
 623         bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
 624                        bo->screen->fd, offset);
 625         if (bo->map == MAP_FAILED) {
 626                 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
 627                         bo->handle, (long long)offset, bo->size);
 628                 abort();
 629         }
 630         VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false));
 631
 632         return bo->map;
 633 }
 634
 635 void *
 636 vc4_bo_map(struct vc4_bo *bo)
 637 {
 638         void *map = vc4_bo_map_unsynchronized(bo);
 639
 640         bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
 641         if (!ok) {
 642                 fprintf(stderr, "BO wait for map failed\n");
 643                 abort();
 644         }
 645
 646         return map;
 647 }
 648
 649 void
 650 vc4_bufmgr_destroy(struct pipe_screen *pscreen)
 651 {
 652         struct vc4_screen *screen = vc4_screen(pscreen);
 653         struct vc4_bo_cache *cache = &screen->bo_cache;
 654
 655         vc4_bo_cache_free_all(cache);
 656
 657         if (dump_stats) {
 658                 fprintf(stderr, "BO stats after screen destroy:\n");
 659                 vc4_bo_dump_stats(screen);
 660         }
 661 }