src/gallium/drivers/vc4/vc4_bufmgr.c

   1 /*
   2  * Copyright © 2014-2015 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <errno.h>
  25 #include <err.h>
  26 #include <sys/mman.h>
  27 #include <fcntl.h>
  28 #include <xf86drm.h>
  29 #include <xf86drmMode.h>
  30
  31 #include "util/u_hash_table.h"
  32 #include "util/u_memory.h"
  33 #include "util/ralloc.h"
  34
  35 #include "vc4_context.h"
  36 #include "vc4_screen.h"
  37
  38 static bool dump_stats = false;
  39
  40 static void
  41 vc4_bo_cache_free_all(struct vc4_bo_cache *cache);
  42
  43 static void
  44 vc4_bo_dump_stats(struct vc4_screen *screen)
  45 {
  46         struct vc4_bo_cache *cache = &screen->bo_cache;
  47
  48         fprintf(stderr, "  BOs allocated:   %d\n", screen->bo_count);
  49         fprintf(stderr, "  BOs size:        %dkb\n", screen->bo_size / 1024);
  50         fprintf(stderr, "  BOs cached:      %d\n", cache->bo_count);
  51         fprintf(stderr, "  BOs cached size: %dkb\n", cache->bo_size / 1024);
  52
  53         if (!list_empty(&cache->time_list)) {
  54                 struct vc4_bo *first = LIST_ENTRY(struct vc4_bo,
  55                                                   cache->time_list.next,
  56                                                   time_list);
  57                 struct vc4_bo *last = LIST_ENTRY(struct vc4_bo,
  58                                                   cache->time_list.prev,
  59                                                   time_list);
  60
  61                 fprintf(stderr, "  oldest cache time: %ld\n",
  62                         (long)first->free_time);
  63                 fprintf(stderr, "  newest cache time: %ld\n",
  64                         (long)last->free_time);
  65
  66                 struct timespec time;
  67                 clock_gettime(CLOCK_MONOTONIC, &time);
  68                 fprintf(stderr, "  now:               %ld\n",
  69                         time.tv_sec);
  70         }
  71 }
  72
  73 static void
  74 vc4_bo_remove_from_cache(struct vc4_bo_cache *cache, struct vc4_bo *bo)
  75 {
  76         list_del(&bo->time_list);
  77         list_del(&bo->size_list);
  78         cache->bo_count--;
  79         cache->bo_size -= bo->size;
  80 }
  81
  82 static struct vc4_bo *
  83 vc4_bo_from_cache(struct vc4_screen *screen, uint32_t size, const char *name)
  84 {
  85         struct vc4_bo_cache *cache = &screen->bo_cache;
  86         uint32_t page_index = size / 4096 - 1;
  87
  88         if (cache->size_list_size <= page_index)
  89                 return NULL;
  90
  91         struct vc4_bo *bo = NULL;
  92         pipe_mutex_lock(cache->lock);
  93         if (!list_empty(&cache->size_list[page_index])) {
  94                 bo = LIST_ENTRY(struct vc4_bo, cache->size_list[page_index].next,
  95                                 size_list);
  96
  97                 /* Check that the BO has gone idle.  If not, then we want to
  98                  * allocate something new instead, since we assume that the
  99                  * user will proceed to CPU map it and fill it with stuff.
 100                  */
 101                 if (!vc4_bo_wait(bo, 0, NULL)) {
 102                         pipe_mutex_unlock(cache->lock);
 103                         return NULL;
 104                 }
 105
 106                 pipe_reference_init(&bo->reference, 1);
 107                 vc4_bo_remove_from_cache(cache, bo);
 108
 109                 bo->name = name;
 110         }
 111         pipe_mutex_unlock(cache->lock);
 112         return bo;
 113 }
 114
 115 struct vc4_bo *
 116 vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name)
 117 {
 118         struct vc4_bo *bo;
 119         int ret;
 120
 121         size = align(size, 4096);
 122
 123         bo = vc4_bo_from_cache(screen, size, name);
 124         if (bo) {
 125                 if (dump_stats) {
 126                         fprintf(stderr, "Allocated %s %dkb from cache:\n",
 127                                 name, size / 1024);
 128                         vc4_bo_dump_stats(screen);
 129                 }
 130                 return bo;
 131         }
 132
 133         bo = CALLOC_STRUCT(vc4_bo);
 134         if (!bo)
 135                 return NULL;
 136
 137         pipe_reference_init(&bo->reference, 1);
 138         bo->screen = screen;
 139         bo->size = size;
 140         bo->name = name;
 141         bo->private = true;
 142
 143         bool cleared_and_retried = false;
 144 retry:
 145         if (!using_vc4_simulator) {
 146                 struct drm_vc4_create_bo create;
 147                 memset(&create, 0, sizeof(create));
 148
 149                 create.size = size;
 150
 151                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_CREATE_BO, &create);
 152                 bo->handle = create.handle;
 153         } else {
 154                 struct drm_mode_create_dumb create;
 155                 memset(&create, 0, sizeof(create));
 156
 157                 create.width = 128;
 158                 create.bpp = 8;
 159                 create.height = (size + 127) / 128;
 160
 161                 ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
 162                 bo->handle = create.handle;
 163                 assert(create.size >= size);
 164         }
 165         if (ret != 0) {
 166                 if (!list_empty(&screen->bo_cache.time_list) &&
 167                     !cleared_and_retried) {
 168                         cleared_and_retried = true;
 169                         vc4_bo_cache_free_all(&screen->bo_cache);
 170                         goto retry;
 171                 }
 172
 173                 free(bo);
 174                 return NULL;
 175         }
 176
 177         screen->bo_count++;
 178         screen->bo_size += bo->size;
 179         if (dump_stats) {
 180                 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
 181                 vc4_bo_dump_stats(screen);
 182         }
 183
 184         return bo;
 185 }
 186
 187 void
 188 vc4_bo_last_unreference(struct vc4_bo *bo)
 189 {
 190         struct vc4_screen *screen = bo->screen;
 191
 192         struct timespec time;
 193         clock_gettime(CLOCK_MONOTONIC, &time);
 194         pipe_mutex_lock(screen->bo_cache.lock);
 195         vc4_bo_last_unreference_locked_timed(bo, time.tv_sec);
 196         pipe_mutex_unlock(screen->bo_cache.lock);
 197 }
 198
 199 static void
 200 vc4_bo_free(struct vc4_bo *bo)
 201 {
 202         struct vc4_screen *screen = bo->screen;
 203
 204         if (bo->map) {
 205 #ifdef USE_VC4_SIMULATOR
 206                 if (bo->simulator_winsys_map) {
 207                         free(bo->map);
 208                         bo->map = bo->simulator_winsys_map;
 209                 }
 210 #endif
 211                 munmap(bo->map, bo->size);
 212         }
 213
 214         struct drm_gem_close c;
 215         memset(&c, 0, sizeof(c));
 216         c.handle = bo->handle;
 217         int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
 218         if (ret != 0)
 219                 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
 220
 221         screen->bo_count--;
 222         screen->bo_size -= bo->size;
 223
 224         if (dump_stats) {
 225                 fprintf(stderr, "Freed %s%s%dkb:\n",
 226                         bo->name ? bo->name : "",
 227                         bo->name ? " " : "",
 228                         bo->size / 1024);
 229                 vc4_bo_dump_stats(screen);
 230         }
 231
 232         free(bo);
 233 }
 234
 235 static void
 236 free_stale_bos(struct vc4_screen *screen, time_t time)
 237 {
 238         struct vc4_bo_cache *cache = &screen->bo_cache;
 239         bool freed_any = false;
 240
 241         list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
 242                                  time_list) {
 243                 if (dump_stats && !freed_any) {
 244                         fprintf(stderr, "Freeing stale BOs:\n");
 245                         vc4_bo_dump_stats(screen);
 246                         freed_any = true;
 247                 }
 248
 249                 /* If it's more than a second old, free it. */
 250                 if (time - bo->free_time > 2) {
 251                         vc4_bo_remove_from_cache(cache, bo);
 252                         vc4_bo_free(bo);
 253                 } else {
 254                         break;
 255                 }
 256         }
 257
 258         if (dump_stats && freed_any) {
 259                 fprintf(stderr, "Freed stale BOs:\n");
 260                 vc4_bo_dump_stats(screen);
 261         }
 262 }
 263
 264 static void
 265 vc4_bo_cache_free_all(struct vc4_bo_cache *cache)
 266 {
 267         pipe_mutex_lock(cache->lock);
 268         list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
 269                                  time_list) {
 270                 vc4_bo_remove_from_cache(cache, bo);
 271                 vc4_bo_free(bo);
 272         }
 273         pipe_mutex_unlock(cache->lock);
 274 }
 275
 276 void
 277 vc4_bo_last_unreference_locked_timed(struct vc4_bo *bo, time_t time)
 278 {
 279         struct vc4_screen *screen = bo->screen;
 280         struct vc4_bo_cache *cache = &screen->bo_cache;
 281         uint32_t page_index = bo->size / 4096 - 1;
 282
 283         if (!bo->private) {
 284                 vc4_bo_free(bo);
 285                 return;
 286         }
 287
 288         if (cache->size_list_size <= page_index) {
 289                 struct list_head *new_list =
 290                         ralloc_array(screen, struct list_head, page_index + 1);
 291
 292                 /* Move old list contents over (since the array has moved, and
 293                  * therefore the pointers to the list heads have to change).
 294                  */
 295                 for (int i = 0; i < cache->size_list_size; i++) {
 296                         struct list_head *old_head = &cache->size_list[i];
 297                         if (list_empty(old_head))
 298                                 list_inithead(&new_list[i]);
 299                         else {
 300                                 new_list[i].next = old_head->next;
 301                                 new_list[i].prev = old_head->prev;
 302                                 new_list[i].next->prev = &new_list[i];
 303                                 new_list[i].prev->next = &new_list[i];
 304                         }
 305                 }
 306                 for (int i = cache->size_list_size; i < page_index + 1; i++)
 307                         list_inithead(&new_list[i]);
 308
 309                 cache->size_list = new_list;
 310                 cache->size_list_size = page_index + 1;
 311         }
 312
 313         bo->free_time = time;
 314         list_addtail(&bo->size_list, &cache->size_list[page_index]);
 315         list_addtail(&bo->time_list, &cache->time_list);
 316         cache->bo_count++;
 317         cache->bo_size += bo->size;
 318         if (dump_stats) {
 319                 fprintf(stderr, "Freed %s %dkb to cache:\n",
 320                         bo->name, bo->size / 1024);
 321                 vc4_bo_dump_stats(screen);
 322         }
 323         bo->name = NULL;
 324
 325         free_stale_bos(screen, time);
 326 }
 327
 328 static struct vc4_bo *
 329 vc4_bo_open_handle(struct vc4_screen *screen,
 330                    uint32_t winsys_stride,
 331                    uint32_t handle, uint32_t size)
 332 {
 333         struct vc4_bo *bo;
 334
 335         assert(size);
 336
 337         pipe_mutex_lock(screen->bo_handles_mutex);
 338
 339         bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
 340         if (bo) {
 341                 pipe_reference(NULL, &bo->reference);
 342                 goto done;
 343         }
 344
 345         bo = CALLOC_STRUCT(vc4_bo);
 346         pipe_reference_init(&bo->reference, 1);
 347         bo->screen = screen;
 348         bo->handle = handle;
 349         bo->size = size;
 350         bo->name = "winsys";
 351         bo->private = false;
 352
 353 #ifdef USE_VC4_SIMULATOR
 354         vc4_bo_map(bo);
 355         bo->simulator_winsys_map = bo->map;
 356         bo->simulator_winsys_stride = winsys_stride;
 357         bo->map = malloc(bo->size);
 358 #endif
 359
 360         util_hash_table_set(screen->bo_handles, (void *)(uintptr_t)handle, bo);
 361
 362 done:
 363         pipe_mutex_unlock(screen->bo_handles_mutex);
 364         return bo;
 365 }
 366
 367 struct vc4_bo *
 368 vc4_bo_open_name(struct vc4_screen *screen, uint32_t name,
 369                  uint32_t winsys_stride)
 370 {
 371         struct drm_gem_open o = {
 372                 .name = name
 373         };
 374         int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
 375         if (ret) {
 376                 fprintf(stderr, "Failed to open bo %d: %s\n",
 377                         name, strerror(errno));
 378                 return NULL;
 379         }
 380
 381         return vc4_bo_open_handle(screen, winsys_stride, o.handle, o.size);
 382 }
 383
 384 struct vc4_bo *
 385 vc4_bo_open_dmabuf(struct vc4_screen *screen, int fd, uint32_t winsys_stride)
 386 {
 387         uint32_t handle;
 388         int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
 389         int size;
 390         if (ret) {
 391                 fprintf(stderr, "Failed to get vc4 handle for dmabuf %d\n", fd);
 392                 return NULL;
 393         }
 394
 395         /* Determine the size of the bo we were handed. */
 396         size = lseek(fd, 0, SEEK_END);
 397         if (size == -1) {
 398                 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
 399                 return NULL;
 400         }
 401
 402         return vc4_bo_open_handle(screen, winsys_stride, handle, size);
 403 }
 404
 405 int
 406 vc4_bo_get_dmabuf(struct vc4_bo *bo)
 407 {
 408         int fd;
 409         int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
 410                                      O_CLOEXEC, &fd);
 411         if (ret != 0) {
 412                 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
 413                         bo->handle);
 414                 return -1;
 415         }
 416
 417         pipe_mutex_lock(bo->screen->bo_handles_mutex);
 418         bo->private = false;
 419         util_hash_table_set(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
 420         pipe_mutex_unlock(bo->screen->bo_handles_mutex);
 421
 422         return fd;
 423 }
 424
 425 struct vc4_bo *
 426 vc4_bo_alloc_shader(struct vc4_screen *screen, const void *data, uint32_t size)
 427 {
 428         struct vc4_bo *bo;
 429         int ret;
 430
 431         bo = CALLOC_STRUCT(vc4_bo);
 432         if (!bo)
 433                 return NULL;
 434
 435         pipe_reference_init(&bo->reference, 1);
 436         bo->screen = screen;
 437         bo->size = align(size, 4096);
 438         bo->name = "code";
 439         bo->private = false; /* Make sure it doesn't go back to the cache. */
 440
 441         if (!using_vc4_simulator) {
 442                 struct drm_vc4_create_shader_bo create = {
 443                         .size = size,
 444                         .data = (uintptr_t)data,
 445                 };
 446
 447                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_CREATE_SHADER_BO,
 448                                &create);
 449                 bo->handle = create.handle;
 450         } else {
 451                 struct drm_mode_create_dumb create;
 452                 memset(&create, 0, sizeof(create));
 453
 454                 create.width = 128;
 455                 create.bpp = 8;
 456                 create.height = (size + 127) / 128;
 457
 458                 ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
 459                 bo->handle = create.handle;
 460                 assert(create.size >= size);
 461
 462                 vc4_bo_map(bo);
 463                 memcpy(bo->map, data, size);
 464         }
 465         if (ret != 0) {
 466                 fprintf(stderr, "create shader ioctl failure\n");
 467                 abort();
 468         }
 469
 470         screen->bo_count++;
 471         screen->bo_size += bo->size;
 472         if (dump_stats) {
 473                 fprintf(stderr, "Allocated shader %dkb:\n", bo->size / 1024);
 474                 vc4_bo_dump_stats(screen);
 475         }
 476
 477         return bo;
 478 }
 479
 480 bool
 481 vc4_bo_flink(struct vc4_bo *bo, uint32_t *name)
 482 {
 483         struct drm_gem_flink flink = {
 484                 .handle = bo->handle,
 485         };
 486         int ret = drmIoctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
 487         if (ret) {
 488                 fprintf(stderr, "Failed to flink bo %d: %s\n",
 489                         bo->handle, strerror(errno));
 490                 free(bo);
 491                 return false;
 492         }
 493
 494         bo->private = false;
 495         *name = flink.name;
 496
 497         return true;
 498 }
 499
 500 static int vc4_wait_seqno_ioctl(int fd, uint64_t seqno, uint64_t timeout_ns)
 501 {
 502         if (using_vc4_simulator)
 503                 return 0;
 504
 505         struct drm_vc4_wait_seqno wait = {
 506                 .seqno = seqno,
 507                 .timeout_ns = timeout_ns,
 508         };
 509         int ret = drmIoctl(fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait);
 510         if (ret == -1)
 511                 return -errno;
 512         else
 513                 return 0;
 514
 515 }
 516
 517 bool
 518 vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns,
 519                const char *reason)
 520 {
 521         if (screen->finished_seqno >= seqno)
 522                 return true;
 523
 524         if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
 525                 if (vc4_wait_seqno_ioctl(screen->fd, seqno, 0) == -ETIME) {
 526                         fprintf(stderr, "Blocking on seqno %lld for %s\n",
 527                                 (long long)seqno, reason);
 528                 }
 529         }
 530
 531         int ret = vc4_wait_seqno_ioctl(screen->fd, seqno, timeout_ns);
 532         if (ret) {
 533                 if (ret != -ETIME) {
 534                         fprintf(stderr, "wait failed: %d\n", ret);
 535                         abort();
 536                 }
 537
 538                 return false;
 539         }
 540
 541         screen->finished_seqno = seqno;
 542         return true;
 543 }
 544
 545 static int vc4_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
 546 {
 547         if (using_vc4_simulator)
 548                 return 0;
 549
 550         struct drm_vc4_wait_bo wait = {
 551                 .handle = handle,
 552                 .timeout_ns = timeout_ns,
 553         };
 554         int ret = drmIoctl(fd, DRM_IOCTL_VC4_WAIT_BO, &wait);
 555         if (ret == -1)
 556                 return -errno;
 557         else
 558                 return 0;
 559
 560 }
 561
 562 bool
 563 vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns, const char *reason)
 564 {
 565         struct vc4_screen *screen = bo->screen;
 566
 567         if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
 568                 if (vc4_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
 569                         fprintf(stderr, "Blocking on %s BO for %s\n",
 570                                 bo->name, reason);
 571                 }
 572         }
 573
 574         int ret = vc4_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
 575         if (ret) {
 576                 if (ret != -ETIME) {
 577                         fprintf(stderr, "wait failed: %d\n", ret);
 578                         abort();
 579                 }
 580
 581                 return false;
 582         }
 583
 584         return true;
 585 }
 586
 587 void *
 588 vc4_bo_map_unsynchronized(struct vc4_bo *bo)
 589 {
 590         uint64_t offset;
 591         int ret;
 592
 593         if (bo->map)
 594                 return bo->map;
 595
 596         if (!using_vc4_simulator) {
 597                 struct drm_vc4_mmap_bo map;
 598                 memset(&map, 0, sizeof(map));
 599                 map.handle = bo->handle;
 600                 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_VC4_MMAP_BO, &map);
 601                 offset = map.offset;
 602         } else {
 603                 struct drm_mode_map_dumb map;
 604                 memset(&map, 0, sizeof(map));
 605                 map.handle = bo->handle;
 606                 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_MODE_MAP_DUMB, &map);
 607                 offset = map.offset;
 608         }
 609         if (ret != 0) {
 610                 fprintf(stderr, "map ioctl failure\n");
 611                 abort();
 612         }
 613
 614         bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
 615                        bo->screen->fd, offset);
 616         if (bo->map == MAP_FAILED) {
 617                 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
 618                         bo->handle, (long long)offset, bo->size);
 619                 abort();
 620         }
 621
 622         return bo->map;
 623 }
 624
 625 void *
 626 vc4_bo_map(struct vc4_bo *bo)
 627 {
 628         void *map = vc4_bo_map_unsynchronized(bo);
 629
 630         bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
 631         if (!ok) {
 632                 fprintf(stderr, "BO wait for map failed\n");
 633                 abort();
 634         }
 635
 636         return map;
 637 }
 638
 639 void
 640 vc4_bufmgr_destroy(struct pipe_screen *pscreen)
 641 {
 642         struct vc4_screen *screen = vc4_screen(pscreen);
 643         struct vc4_bo_cache *cache = &screen->bo_cache;
 644
 645         vc4_bo_cache_free_all(cache);
 646
 647         if (dump_stats) {
 648                 fprintf(stderr, "BO stats after screen destroy:\n");
 649                 vc4_bo_dump_stats(screen);
 650         }
 651 }