src/gallium/drivers/vc4/vc4_bufmgr.c

   1 /*
   2  * Copyright © 2014-2015 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <errno.h>
  25 #include <err.h>
  26 #include <sys/mman.h>
  27 #include <fcntl.h>
  28 #include <xf86drm.h>
  29 #include <xf86drmMode.h>
  30
  31 #include "util/u_memory.h"
  32 #include "util/ralloc.h"
  33
  34 #include "vc4_context.h"
  35 #include "vc4_screen.h"
  36
  37 static bool dump_stats = false;
  38
  39 static void
  40 vc4_bo_dump_stats(struct vc4_screen *screen)
  41 {
  42         struct vc4_bo_cache *cache = &screen->bo_cache;
  43
  44         fprintf(stderr, "  BOs allocated:   %d\n", screen->bo_count);
  45         fprintf(stderr, "  BOs size:        %dkb\n", screen->bo_size / 1024);
  46         fprintf(stderr, "  BOs cached:      %d\n", cache->bo_count);
  47         fprintf(stderr, "  BOs cached size: %dkb\n", cache->bo_size / 1024);
  48
  49         if (!list_empty(&cache->time_list)) {
  50                 struct vc4_bo *first = LIST_ENTRY(struct vc4_bo,
  51                                                   cache->time_list.next,
  52                                                   time_list);
  53                 struct vc4_bo *last = LIST_ENTRY(struct vc4_bo,
  54                                                   cache->time_list.prev,
  55                                                   time_list);
  56
  57                 fprintf(stderr, "  oldest cache time: %ld\n",
  58                         (long)first->free_time);
  59                 fprintf(stderr, "  newest cache time: %ld\n",
  60                         (long)last->free_time);
  61
  62                 struct timespec time;
  63                 clock_gettime(CLOCK_MONOTONIC, &time);
  64                 fprintf(stderr, "  now:               %ld\n",
  65                         time.tv_sec);
  66         }
  67 }
  68
  69 static void
  70 vc4_bo_remove_from_cache(struct vc4_bo_cache *cache, struct vc4_bo *bo)
  71 {
  72         list_del(&bo->time_list);
  73         list_del(&bo->size_list);
  74         cache->bo_count--;
  75         cache->bo_size -= bo->size;
  76 }
  77
  78 static struct vc4_bo *
  79 vc4_bo_from_cache(struct vc4_screen *screen, uint32_t size, const char *name)
  80 {
  81         struct vc4_bo_cache *cache = &screen->bo_cache;
  82         uint32_t page_index = size / 4096 - 1;
  83
  84         if (cache->size_list_size <= page_index)
  85                 return NULL;
  86
  87         struct vc4_bo *bo = NULL;
  88         pipe_mutex_lock(cache->lock);
  89         if (!list_empty(&cache->size_list[page_index])) {
  90                 bo = LIST_ENTRY(struct vc4_bo, cache->size_list[page_index].next,
  91                                 size_list);
  92
  93                 /* Check that the BO has gone idle.  If not, then we want to
  94                  * allocate something new instead, since we assume that the
  95                  * user will proceed to CPU map it and fill it with stuff.
  96                  */
  97                 if (!vc4_bo_wait(bo, 0, NULL)) {
  98                         pipe_mutex_unlock(cache->lock);
  99                         return NULL;
 100                 }
 101
 102                 pipe_reference_init(&bo->reference, 1);
 103                 vc4_bo_remove_from_cache(cache, bo);
 104
 105                 bo->name = name;
 106         }
 107         pipe_mutex_unlock(cache->lock);
 108         return bo;
 109 }
 110
 111 struct vc4_bo *
 112 vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name)
 113 {
 114         struct vc4_bo *bo;
 115         int ret;
 116
 117         size = align(size, 4096);
 118
 119         bo = vc4_bo_from_cache(screen, size, name);
 120         if (bo) {
 121                 if (dump_stats) {
 122                         fprintf(stderr, "Allocated %s %dkb from cache:\n",
 123                                 name, size / 1024);
 124                         vc4_bo_dump_stats(screen);
 125                 }
 126                 return bo;
 127         }
 128
 129         bo = CALLOC_STRUCT(vc4_bo);
 130         if (!bo)
 131                 return NULL;
 132
 133         pipe_reference_init(&bo->reference, 1);
 134         bo->screen = screen;
 135         bo->size = size;
 136         bo->name = name;
 137         bo->private = true;
 138
 139         if (!using_vc4_simulator) {
 140                 struct drm_vc4_create_bo create;
 141                 memset(&create, 0, sizeof(create));
 142
 143                 create.size = size;
 144
 145                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_CREATE_BO, &create);
 146                 bo->handle = create.handle;
 147         } else {
 148                 struct drm_mode_create_dumb create;
 149                 memset(&create, 0, sizeof(create));
 150
 151                 create.width = 128;
 152                 create.bpp = 8;
 153                 create.height = (size + 127) / 128;
 154
 155                 ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
 156                 bo->handle = create.handle;
 157                 assert(create.size >= size);
 158         }
 159         if (ret != 0) {
 160                 fprintf(stderr, "create ioctl failure\n");
 161                 abort();
 162         }
 163
 164         screen->bo_count++;
 165         screen->bo_size += bo->size;
 166         if (dump_stats) {
 167                 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
 168                 vc4_bo_dump_stats(screen);
 169         }
 170
 171         return bo;
 172 }
 173
 174 void
 175 vc4_bo_last_unreference(struct vc4_bo *bo)
 176 {
 177         struct vc4_screen *screen = bo->screen;
 178
 179         struct timespec time;
 180         clock_gettime(CLOCK_MONOTONIC, &time);
 181         pipe_mutex_lock(screen->bo_cache.lock);
 182         vc4_bo_last_unreference_locked_timed(bo, time.tv_sec);
 183         pipe_mutex_unlock(screen->bo_cache.lock);
 184 }
 185
 186 static void
 187 vc4_bo_free(struct vc4_bo *bo)
 188 {
 189         struct vc4_screen *screen = bo->screen;
 190
 191         if (bo->map) {
 192 #ifdef USE_VC4_SIMULATOR
 193                 if (bo->simulator_winsys_map) {
 194                         free(bo->map);
 195                         bo->map = bo->simulator_winsys_map;
 196                 }
 197 #endif
 198                 munmap(bo->map, bo->size);
 199         }
 200
 201         struct drm_gem_close c;
 202         memset(&c, 0, sizeof(c));
 203         c.handle = bo->handle;
 204         int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
 205         if (ret != 0)
 206                 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
 207
 208         screen->bo_count--;
 209         screen->bo_size -= bo->size;
 210
 211         if (dump_stats) {
 212                 fprintf(stderr, "Freed %s%s%dkb:\n",
 213                         bo->name ? bo->name : "",
 214                         bo->name ? " " : "",
 215                         bo->size / 1024);
 216                 vc4_bo_dump_stats(screen);
 217         }
 218
 219         free(bo);
 220 }
 221
 222 static void
 223 free_stale_bos(struct vc4_screen *screen, time_t time)
 224 {
 225         struct vc4_bo_cache *cache = &screen->bo_cache;
 226         bool freed_any = false;
 227
 228         list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
 229                                  time_list) {
 230                 if (dump_stats && !freed_any) {
 231                         fprintf(stderr, "Freeing stale BOs:\n");
 232                         vc4_bo_dump_stats(screen);
 233                         freed_any = true;
 234                 }
 235
 236                 /* If it's more than a second old, free it. */
 237                 if (time - bo->free_time > 2) {
 238                         vc4_bo_remove_from_cache(cache, bo);
 239                         vc4_bo_free(bo);
 240                 } else {
 241                         break;
 242                 }
 243         }
 244
 245         if (dump_stats && freed_any) {
 246                 fprintf(stderr, "Freed stale BOs:\n");
 247                 vc4_bo_dump_stats(screen);
 248         }
 249 }
 250
 251 void
 252 vc4_bo_last_unreference_locked_timed(struct vc4_bo *bo, time_t time)
 253 {
 254         struct vc4_screen *screen = bo->screen;
 255         struct vc4_bo_cache *cache = &screen->bo_cache;
 256         uint32_t page_index = bo->size / 4096 - 1;
 257
 258         if (!bo->private) {
 259                 vc4_bo_free(bo);
 260                 return;
 261         }
 262
 263         if (cache->size_list_size <= page_index) {
 264                 struct list_head *new_list =
 265                         ralloc_array(screen, struct list_head, page_index + 1);
 266
 267                 /* Move old list contents over (since the array has moved, and
 268                  * therefore the pointers to the list heads have to change).
 269                  */
 270                 for (int i = 0; i < cache->size_list_size; i++) {
 271                         struct list_head *old_head = &cache->size_list[i];
 272                         if (list_empty(old_head))
 273                                 list_inithead(&new_list[i]);
 274                         else {
 275                                 new_list[i].next = old_head->next;
 276                                 new_list[i].prev = old_head->prev;
 277                                 new_list[i].next->prev = &new_list[i];
 278                                 new_list[i].prev->next = &new_list[i];
 279                         }
 280                 }
 281                 for (int i = cache->size_list_size; i < page_index + 1; i++)
 282                         list_inithead(&new_list[i]);
 283
 284                 cache->size_list = new_list;
 285                 cache->size_list_size = page_index + 1;
 286         }
 287
 288         bo->free_time = time;
 289         list_addtail(&bo->size_list, &cache->size_list[page_index]);
 290         list_addtail(&bo->time_list, &cache->time_list);
 291         cache->bo_count++;
 292         cache->bo_size += bo->size;
 293         if (dump_stats) {
 294                 fprintf(stderr, "Freed %s %dkb to cache:\n",
 295                         bo->name, bo->size / 1024);
 296                 vc4_bo_dump_stats(screen);
 297         }
 298         bo->name = NULL;
 299
 300         free_stale_bos(screen, time);
 301 }
 302
 303 static struct vc4_bo *
 304 vc4_bo_open_handle(struct vc4_screen *screen,
 305                    uint32_t winsys_stride,
 306                    uint32_t handle, uint32_t size)
 307 {
 308         struct vc4_bo *bo = CALLOC_STRUCT(vc4_bo);
 309
 310         assert(size);
 311
 312         pipe_reference_init(&bo->reference, 1);
 313         bo->screen = screen;
 314         bo->handle = handle;
 315         bo->size = size;
 316         bo->name = "winsys";
 317         bo->private = false;
 318
 319 #ifdef USE_VC4_SIMULATOR
 320         vc4_bo_map(bo);
 321         bo->simulator_winsys_map = bo->map;
 322         bo->simulator_winsys_stride = winsys_stride;
 323         bo->map = malloc(bo->size);
 324 #endif
 325
 326         return bo;
 327 }
 328
 329 struct vc4_bo *
 330 vc4_bo_open_name(struct vc4_screen *screen, uint32_t name,
 331                  uint32_t winsys_stride)
 332 {
 333         struct drm_gem_open o = {
 334                 .name = name
 335         };
 336         int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
 337         if (ret) {
 338                 fprintf(stderr, "Failed to open bo %d: %s\n",
 339                         name, strerror(errno));
 340                 return NULL;
 341         }
 342
 343         return vc4_bo_open_handle(screen, winsys_stride, o.handle, o.size);
 344 }
 345
 346 struct vc4_bo *
 347 vc4_bo_open_dmabuf(struct vc4_screen *screen, int fd, uint32_t winsys_stride)
 348 {
 349         uint32_t handle;
 350         int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
 351         int size;
 352         if (ret) {
 353                 fprintf(stderr, "Failed to get vc4 handle for dmabuf %d\n", fd);
 354                 return NULL;
 355         }
 356
 357         /* Determine the size of the bo we were handed. */
 358         size = lseek(fd, 0, SEEK_END);
 359         if (size == -1) {
 360                 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
 361                 return NULL;
 362         }
 363
 364         return vc4_bo_open_handle(screen, winsys_stride, handle, size);
 365 }
 366
 367 int
 368 vc4_bo_get_dmabuf(struct vc4_bo *bo)
 369 {
 370         int fd;
 371         int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
 372                                      O_CLOEXEC, &fd);
 373         if (ret != 0) {
 374                 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
 375                         bo->handle);
 376                 return -1;
 377         }
 378         bo->private = false;
 379
 380         return fd;
 381 }
 382
 383 struct vc4_bo *
 384 vc4_bo_alloc_shader(struct vc4_screen *screen, const void *data, uint32_t size)
 385 {
 386         struct vc4_bo *bo;
 387         int ret;
 388
 389         bo = CALLOC_STRUCT(vc4_bo);
 390         if (!bo)
 391                 return NULL;
 392
 393         pipe_reference_init(&bo->reference, 1);
 394         bo->screen = screen;
 395         bo->size = align(size, 4096);
 396         bo->name = "code";
 397         bo->private = false; /* Make sure it doesn't go back to the cache. */
 398
 399         if (!using_vc4_simulator) {
 400                 struct drm_vc4_create_shader_bo create = {
 401                         .size = size,
 402                         .data = (uintptr_t)data,
 403                 };
 404
 405                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_CREATE_SHADER_BO,
 406                                &create);
 407                 bo->handle = create.handle;
 408         } else {
 409                 struct drm_mode_create_dumb create;
 410                 memset(&create, 0, sizeof(create));
 411
 412                 create.width = 128;
 413                 create.bpp = 8;
 414                 create.height = (size + 127) / 128;
 415
 416                 ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
 417                 bo->handle = create.handle;
 418                 assert(create.size >= size);
 419
 420                 vc4_bo_map(bo);
 421                 memcpy(bo->map, data, size);
 422         }
 423         if (ret != 0) {
 424                 fprintf(stderr, "create shader ioctl failure\n");
 425                 abort();
 426         }
 427
 428         screen->bo_count++;
 429         screen->bo_size += bo->size;
 430         if (dump_stats) {
 431                 fprintf(stderr, "Allocated shader %dkb:\n", size / 1024);
 432                 vc4_bo_dump_stats(screen);
 433         }
 434
 435         return bo;
 436 }
 437
 438 bool
 439 vc4_bo_flink(struct vc4_bo *bo, uint32_t *name)
 440 {
 441         struct drm_gem_flink flink = {
 442                 .handle = bo->handle,
 443         };
 444         int ret = drmIoctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
 445         if (ret) {
 446                 fprintf(stderr, "Failed to flink bo %d: %s\n",
 447                         bo->handle, strerror(errno));
 448                 free(bo);
 449                 return false;
 450         }
 451
 452         bo->private = false;
 453         *name = flink.name;
 454
 455         return true;
 456 }
 457
 458 static int vc4_wait_seqno_ioctl(int fd, uint64_t seqno, uint64_t timeout_ns)
 459 {
 460         if (using_vc4_simulator)
 461                 return 0;
 462
 463         struct drm_vc4_wait_seqno wait = {
 464                 .seqno = seqno,
 465                 .timeout_ns = timeout_ns,
 466         };
 467         int ret = drmIoctl(fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait);
 468         if (ret == -1)
 469                 return -errno;
 470         else
 471                 return 0;
 472
 473 }
 474
 475 bool
 476 vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns,
 477                const char *reason)
 478 {
 479         if (screen->finished_seqno >= seqno)
 480                 return true;
 481
 482         if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
 483                 if (vc4_wait_seqno_ioctl(screen->fd, seqno, 0) == -ETIME) {
 484                         fprintf(stderr, "Blocking on seqno %lld for %s\n",
 485                                 (long long)seqno, reason);
 486                 }
 487         }
 488
 489         int ret = vc4_wait_seqno_ioctl(screen->fd, seqno, timeout_ns);
 490         if (ret) {
 491                 if (ret != -ETIME) {
 492                         fprintf(stderr, "wait failed: %d\n", ret);
 493                         abort();
 494                 }
 495
 496                 return false;
 497         }
 498
 499         screen->finished_seqno = seqno;
 500         return true;
 501 }
 502
 503 static int vc4_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
 504 {
 505         if (using_vc4_simulator)
 506                 return 0;
 507
 508         struct drm_vc4_wait_bo wait = {
 509                 .handle = handle,
 510                 .timeout_ns = timeout_ns,
 511         };
 512         int ret = drmIoctl(fd, DRM_IOCTL_VC4_WAIT_BO, &wait);
 513         if (ret == -1)
 514                 return -errno;
 515         else
 516                 return 0;
 517
 518 }
 519
 520 bool
 521 vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns, const char *reason)
 522 {
 523         struct vc4_screen *screen = bo->screen;
 524
 525         if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
 526                 if (vc4_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
 527                         fprintf(stderr, "Blocking on %s BO for %s\n",
 528                                 bo->name, reason);
 529                 }
 530         }
 531
 532         int ret = vc4_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
 533         if (ret) {
 534                 if (ret != -ETIME) {
 535                         fprintf(stderr, "wait failed: %d\n", ret);
 536                         abort();
 537                 }
 538
 539                 return false;
 540         }
 541
 542         return true;
 543 }
 544
 545 void *
 546 vc4_bo_map_unsynchronized(struct vc4_bo *bo)
 547 {
 548         uint64_t offset;
 549         int ret;
 550
 551         if (bo->map)
 552                 return bo->map;
 553
 554         if (!using_vc4_simulator) {
 555                 struct drm_vc4_mmap_bo map;
 556                 memset(&map, 0, sizeof(map));
 557                 map.handle = bo->handle;
 558                 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_VC4_MMAP_BO, &map);
 559                 offset = map.offset;
 560         } else {
 561                 struct drm_mode_map_dumb map;
 562                 memset(&map, 0, sizeof(map));
 563                 map.handle = bo->handle;
 564                 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_MODE_MAP_DUMB, &map);
 565                 offset = map.offset;
 566         }
 567         if (ret != 0) {
 568                 fprintf(stderr, "map ioctl failure\n");
 569                 abort();
 570         }
 571
 572         bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
 573                        bo->screen->fd, offset);
 574         if (bo->map == MAP_FAILED) {
 575                 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
 576                         bo->handle, (long long)offset, bo->size);
 577                 abort();
 578         }
 579
 580         return bo->map;
 581 }
 582
 583 void *
 584 vc4_bo_map(struct vc4_bo *bo)
 585 {
 586         void *map = vc4_bo_map_unsynchronized(bo);
 587
 588         bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
 589         if (!ok) {
 590                 fprintf(stderr, "BO wait for map failed\n");
 591                 abort();
 592         }
 593
 594         return map;
 595 }
 596
 597 void
 598 vc4_bufmgr_destroy(struct pipe_screen *pscreen)
 599 {
 600         struct vc4_screen *screen = vc4_screen(pscreen);
 601         struct vc4_bo_cache *cache = &screen->bo_cache;
 602
 603         list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
 604                                  time_list) {
 605                 vc4_bo_remove_from_cache(cache, bo);
 606                 vc4_bo_free(bo);
 607         }
 608
 609         if (dump_stats) {
 610                 fprintf(stderr, "BO stats after screen destroy:\n");
 611                 vc4_bo_dump_stats(screen);
 612         }
 613 }