src/gallium/drivers/vc5/vc5_bufmgr.c

   1 /*
   2  * Copyright © 2014-2017 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <errno.h>
  25 #include <err.h>
  26 #include <sys/mman.h>
  27 #include <fcntl.h>
  28 #include <xf86drm.h>
  29 #include <xf86drmMode.h>
  30
  31 #include "util/u_hash_table.h"
  32 #include "util/u_memory.h"
  33 #include "util/ralloc.h"
  34
  35 #include "vc5_context.h"
  36 #include "vc5_screen.h"
  37
  38 #ifdef HAVE_VALGRIND
  39 #include <valgrind.h>
  40 #include <memcheck.h>
  41 #define VG(x) x
  42 #else
  43 #define VG(x)
  44 #endif
  45
  46 static bool dump_stats = false;
  47
  48 static void
  49 vc5_bo_cache_free_all(struct vc5_bo_cache *cache);
  50
  51 static void
  52 vc5_bo_dump_stats(struct vc5_screen *screen)
  53 {
  54         struct vc5_bo_cache *cache = &screen->bo_cache;
  55
  56         fprintf(stderr, "  BOs allocated:   %d\n", screen->bo_count);
  57         fprintf(stderr, "  BOs size:        %dkb\n", screen->bo_size / 1024);
  58         fprintf(stderr, "  BOs cached:      %d\n", cache->bo_count);
  59         fprintf(stderr, "  BOs cached size: %dkb\n", cache->bo_size / 1024);
  60
  61         if (!list_empty(&cache->time_list)) {
  62                 struct vc5_bo *first = LIST_ENTRY(struct vc5_bo,
  63                                                   cache->time_list.next,
  64                                                   time_list);
  65                 struct vc5_bo *last = LIST_ENTRY(struct vc5_bo,
  66                                                   cache->time_list.prev,
  67                                                   time_list);
  68
  69                 fprintf(stderr, "  oldest cache time: %ld\n",
  70                         (long)first->free_time);
  71                 fprintf(stderr, "  newest cache time: %ld\n",
  72                         (long)last->free_time);
  73
  74                 struct timespec time;
  75                 clock_gettime(CLOCK_MONOTONIC, &time);
  76                 fprintf(stderr, "  now:               %ld\n",
  77                         time.tv_sec);
  78         }
  79 }
  80
  81 static void
  82 vc5_bo_remove_from_cache(struct vc5_bo_cache *cache, struct vc5_bo *bo)
  83 {
  84         list_del(&bo->time_list);
  85         list_del(&bo->size_list);
  86         cache->bo_count--;
  87         cache->bo_size -= bo->size;
  88 }
  89
  90 static struct vc5_bo *
  91 vc5_bo_from_cache(struct vc5_screen *screen, uint32_t size, const char *name)
  92 {
  93         struct vc5_bo_cache *cache = &screen->bo_cache;
  94         uint32_t page_index = size / 4096 - 1;
  95
  96         if (cache->size_list_size <= page_index)
  97                 return NULL;
  98
  99         struct vc5_bo *bo = NULL;
 100         mtx_lock(&cache->lock);
 101         if (!list_empty(&cache->size_list[page_index])) {
 102                 bo = LIST_ENTRY(struct vc5_bo, cache->size_list[page_index].next,
 103                                 size_list);
 104
 105                 /* Check that the BO has gone idle.  If not, then we want to
 106                  * allocate something new instead, since we assume that the
 107                  * user will proceed to CPU map it and fill it with stuff.
 108                  */
 109                 if (!vc5_bo_wait(bo, 0, NULL)) {
 110                         mtx_unlock(&cache->lock);
 111                         return NULL;
 112                 }
 113
 114                 pipe_reference_init(&bo->reference, 1);
 115                 vc5_bo_remove_from_cache(cache, bo);
 116
 117                 bo->name = name;
 118         }
 119         mtx_unlock(&cache->lock);
 120         return bo;
 121 }
 122
 123 struct vc5_bo *
 124 vc5_bo_alloc(struct vc5_screen *screen, uint32_t size, const char *name)
 125 {
 126         struct vc5_bo *bo;
 127         int ret;
 128
 129         size = align(size, 4096);
 130
 131         bo = vc5_bo_from_cache(screen, size, name);
 132         if (bo) {
 133                 if (dump_stats) {
 134                         fprintf(stderr, "Allocated %s %dkb from cache:\n",
 135                                 name, size / 1024);
 136                         vc5_bo_dump_stats(screen);
 137                 }
 138                 return bo;
 139         }
 140
 141         bo = CALLOC_STRUCT(vc5_bo);
 142         if (!bo)
 143                 return NULL;
 144
 145         pipe_reference_init(&bo->reference, 1);
 146         bo->screen = screen;
 147         bo->size = size;
 148         bo->name = name;
 149         bo->private = true;
 150
 151  retry:
 152         ;
 153
 154         bool cleared_and_retried = false;
 155         struct drm_v3d_create_bo create = {
 156                 .size = size
 157         };
 158
 159         ret = vc5_ioctl(screen->fd, DRM_IOCTL_V3D_CREATE_BO, &create);
 160         bo->handle = create.handle;
 161         bo->offset = create.offset;
 162
 163         if (ret != 0) {
 164                 if (!list_empty(&screen->bo_cache.time_list) &&
 165                     !cleared_and_retried) {
 166                         cleared_and_retried = true;
 167                         vc5_bo_cache_free_all(&screen->bo_cache);
 168                         goto retry;
 169                 }
 170
 171                 free(bo);
 172                 return NULL;
 173         }
 174
 175         screen->bo_count++;
 176         screen->bo_size += bo->size;
 177         if (dump_stats) {
 178                 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
 179                 vc5_bo_dump_stats(screen);
 180         }
 181
 182         return bo;
 183 }
 184
 185 void
 186 vc5_bo_last_unreference(struct vc5_bo *bo)
 187 {
 188         struct vc5_screen *screen = bo->screen;
 189
 190         struct timespec time;
 191         clock_gettime(CLOCK_MONOTONIC, &time);
 192         mtx_lock(&screen->bo_cache.lock);
 193         vc5_bo_last_unreference_locked_timed(bo, time.tv_sec);
 194         mtx_unlock(&screen->bo_cache.lock);
 195 }
 196
 197 static void
 198 vc5_bo_free(struct vc5_bo *bo)
 199 {
 200         struct vc5_screen *screen = bo->screen;
 201
 202         if (bo->map) {
 203                 if (using_vc5_simulator && bo->name &&
 204                     strcmp(bo->name, "winsys") == 0) {
 205                         free(bo->map);
 206                 } else {
 207                         munmap(bo->map, bo->size);
 208                         VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
 209                 }
 210         }
 211
 212         struct drm_gem_close c;
 213         memset(&c, 0, sizeof(c));
 214         c.handle = bo->handle;
 215         int ret = vc5_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
 216         if (ret != 0)
 217                 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
 218
 219         screen->bo_count--;
 220         screen->bo_size -= bo->size;
 221
 222         if (dump_stats) {
 223                 fprintf(stderr, "Freed %s%s%dkb:\n",
 224                         bo->name ? bo->name : "",
 225                         bo->name ? " " : "",
 226                         bo->size / 1024);
 227                 vc5_bo_dump_stats(screen);
 228         }
 229
 230         free(bo);
 231 }
 232
 233 static void
 234 free_stale_bos(struct vc5_screen *screen, time_t time)
 235 {
 236         struct vc5_bo_cache *cache = &screen->bo_cache;
 237         bool freed_any = false;
 238
 239         list_for_each_entry_safe(struct vc5_bo, bo, &cache->time_list,
 240                                  time_list) {
 241                 if (dump_stats && !freed_any) {
 242                         fprintf(stderr, "Freeing stale BOs:\n");
 243                         vc5_bo_dump_stats(screen);
 244                         freed_any = true;
 245                 }
 246
 247                 /* If it's more than a second old, free it. */
 248                 if (time - bo->free_time > 2) {
 249                         vc5_bo_remove_from_cache(cache, bo);
 250                         vc5_bo_free(bo);
 251                 } else {
 252                         break;
 253                 }
 254         }
 255
 256         if (dump_stats && freed_any) {
 257                 fprintf(stderr, "Freed stale BOs:\n");
 258                 vc5_bo_dump_stats(screen);
 259         }
 260 }
 261
 262 static void
 263 vc5_bo_cache_free_all(struct vc5_bo_cache *cache)
 264 {
 265         mtx_lock(&cache->lock);
 266         list_for_each_entry_safe(struct vc5_bo, bo, &cache->time_list,
 267                                  time_list) {
 268                 vc5_bo_remove_from_cache(cache, bo);
 269                 vc5_bo_free(bo);
 270         }
 271         mtx_unlock(&cache->lock);
 272 }
 273
 274 void
 275 vc5_bo_last_unreference_locked_timed(struct vc5_bo *bo, time_t time)
 276 {
 277         struct vc5_screen *screen = bo->screen;
 278         struct vc5_bo_cache *cache = &screen->bo_cache;
 279         uint32_t page_index = bo->size / 4096 - 1;
 280
 281         if (!bo->private) {
 282                 vc5_bo_free(bo);
 283                 return;
 284         }
 285
 286         if (cache->size_list_size <= page_index) {
 287                 struct list_head *new_list =
 288                         ralloc_array(screen, struct list_head, page_index + 1);
 289
 290                 /* Move old list contents over (since the array has moved, and
 291                  * therefore the pointers to the list heads have to change).
 292                  */
 293                 for (int i = 0; i < cache->size_list_size; i++) {
 294                         struct list_head *old_head = &cache->size_list[i];
 295                         if (list_empty(old_head))
 296                                 list_inithead(&new_list[i]);
 297                         else {
 298                                 new_list[i].next = old_head->next;
 299                                 new_list[i].prev = old_head->prev;
 300                                 new_list[i].next->prev = &new_list[i];
 301                                 new_list[i].prev->next = &new_list[i];
 302                         }
 303                 }
 304                 for (int i = cache->size_list_size; i < page_index + 1; i++)
 305                         list_inithead(&new_list[i]);
 306
 307                 cache->size_list = new_list;
 308                 cache->size_list_size = page_index + 1;
 309         }
 310
 311         bo->free_time = time;
 312         list_addtail(&bo->size_list, &cache->size_list[page_index]);
 313         list_addtail(&bo->time_list, &cache->time_list);
 314         cache->bo_count++;
 315         cache->bo_size += bo->size;
 316         if (dump_stats) {
 317                 fprintf(stderr, "Freed %s %dkb to cache:\n",
 318                         bo->name, bo->size / 1024);
 319                 vc5_bo_dump_stats(screen);
 320         }
 321         bo->name = NULL;
 322
 323         free_stale_bos(screen, time);
 324 }
 325
 326 static struct vc5_bo *
 327 vc5_bo_open_handle(struct vc5_screen *screen,
 328                    uint32_t winsys_stride,
 329                    uint32_t handle, uint32_t size)
 330 {
 331         struct vc5_bo *bo;
 332
 333         assert(size);
 334
 335         mtx_lock(&screen->bo_handles_mutex);
 336
 337         bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
 338         if (bo) {
 339                 pipe_reference(NULL, &bo->reference);
 340                 goto done;
 341         }
 342
 343         bo = CALLOC_STRUCT(vc5_bo);
 344         pipe_reference_init(&bo->reference, 1);
 345         bo->screen = screen;
 346         bo->handle = handle;
 347         bo->size = size;
 348         bo->name = "winsys";
 349         bo->private = false;
 350
 351 #ifdef USE_VC5_SIMULATOR
 352         vc5_simulator_open_from_handle(screen->fd, winsys_stride,
 353                                        bo->handle, bo->size);
 354         bo->map = malloc(bo->size);
 355 #endif
 356
 357         struct drm_v3d_get_bo_offset get = {
 358                 .handle = handle,
 359         };
 360         int ret = vc5_ioctl(screen->fd, DRM_IOCTL_V3D_GET_BO_OFFSET, &get);
 361         if (ret) {
 362                 fprintf(stderr, "Failed to get BO offset: %s\n",
 363                         strerror(errno));
 364                 free(bo->map);
 365                 free(bo);
 366                 return NULL;
 367         }
 368         bo->offset = get.offset;
 369         assert(bo->offset != 0);
 370
 371         util_hash_table_set(screen->bo_handles, (void *)(uintptr_t)handle, bo);
 372
 373 done:
 374         mtx_unlock(&screen->bo_handles_mutex);
 375         return bo;
 376 }
 377
 378 struct vc5_bo *
 379 vc5_bo_open_name(struct vc5_screen *screen, uint32_t name,
 380                  uint32_t winsys_stride)
 381 {
 382         struct drm_gem_open o = {
 383                 .name = name
 384         };
 385         int ret = vc5_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
 386         if (ret) {
 387                 fprintf(stderr, "Failed to open bo %d: %s\n",
 388                         name, strerror(errno));
 389                 return NULL;
 390         }
 391
 392         return vc5_bo_open_handle(screen, winsys_stride, o.handle, o.size);
 393 }
 394
 395 struct vc5_bo *
 396 vc5_bo_open_dmabuf(struct vc5_screen *screen, int fd, uint32_t winsys_stride)
 397 {
 398         uint32_t handle;
 399         int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
 400         int size;
 401         if (ret) {
 402                 fprintf(stderr, "Failed to get vc5 handle for dmabuf %d\n", fd);
 403                 return NULL;
 404         }
 405
 406         /* Determine the size of the bo we were handed. */
 407         size = lseek(fd, 0, SEEK_END);
 408         if (size == -1) {
 409                 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
 410                 return NULL;
 411         }
 412
 413         return vc5_bo_open_handle(screen, winsys_stride, handle, size);
 414 }
 415
 416 int
 417 vc5_bo_get_dmabuf(struct vc5_bo *bo)
 418 {
 419         int fd;
 420         int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
 421                                      O_CLOEXEC, &fd);
 422         if (ret != 0) {
 423                 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
 424                         bo->handle);
 425                 return -1;
 426         }
 427
 428         mtx_lock(&bo->screen->bo_handles_mutex);
 429         bo->private = false;
 430         util_hash_table_set(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
 431         mtx_unlock(&bo->screen->bo_handles_mutex);
 432
 433         return fd;
 434 }
 435
 436 bool
 437 vc5_bo_flink(struct vc5_bo *bo, uint32_t *name)
 438 {
 439         struct drm_gem_flink flink = {
 440                 .handle = bo->handle,
 441         };
 442         int ret = vc5_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
 443         if (ret) {
 444                 fprintf(stderr, "Failed to flink bo %d: %s\n",
 445                         bo->handle, strerror(errno));
 446                 free(bo);
 447                 return false;
 448         }
 449
 450         bo->private = false;
 451         *name = flink.name;
 452
 453         return true;
 454 }
 455
 456 static int vc5_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
 457 {
 458         struct drm_v3d_wait_bo wait = {
 459                 .handle = handle,
 460                 .timeout_ns = timeout_ns,
 461         };
 462         int ret = vc5_ioctl(fd, DRM_IOCTL_V3D_WAIT_BO, &wait);
 463         if (ret == -1)
 464                 return -errno;
 465         else
 466                 return 0;
 467
 468 }
 469
 470 bool
 471 vc5_bo_wait(struct vc5_bo *bo, uint64_t timeout_ns, const char *reason)
 472 {
 473         struct vc5_screen *screen = bo->screen;
 474
 475         if (unlikely(V3D_DEBUG & V3D_DEBUG_PERF) && timeout_ns && reason) {
 476                 if (vc5_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
 477                         fprintf(stderr, "Blocking on %s BO for %s\n",
 478                                 bo->name, reason);
 479                 }
 480         }
 481
 482         int ret = vc5_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
 483         if (ret) {
 484                 if (ret != -ETIME) {
 485                         fprintf(stderr, "wait failed: %d\n", ret);
 486                         abort();
 487                 }
 488
 489                 return false;
 490         }
 491
 492         return true;
 493 }
 494
 495 void *
 496 vc5_bo_map_unsynchronized(struct vc5_bo *bo)
 497 {
 498         uint64_t offset;
 499         int ret;
 500
 501         if (bo->map)
 502                 return bo->map;
 503
 504         struct drm_v3d_mmap_bo map;
 505         memset(&map, 0, sizeof(map));
 506         map.handle = bo->handle;
 507         ret = vc5_ioctl(bo->screen->fd, DRM_IOCTL_V3D_MMAP_BO, &map);
 508         offset = map.offset;
 509         if (ret != 0) {
 510                 fprintf(stderr, "map ioctl failure\n");
 511                 abort();
 512         }
 513
 514         bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
 515                        bo->screen->fd, offset);
 516         if (bo->map == MAP_FAILED) {
 517                 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
 518                         bo->handle, (long long)offset, bo->size);
 519                 abort();
 520         }
 521         VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false));
 522
 523         return bo->map;
 524 }
 525
 526 void *
 527 vc5_bo_map(struct vc5_bo *bo)
 528 {
 529         void *map = vc5_bo_map_unsynchronized(bo);
 530
 531         bool ok = vc5_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
 532         if (!ok) {
 533                 fprintf(stderr, "BO wait for map failed\n");
 534                 abort();
 535         }
 536
 537         return map;
 538 }
 539
 540 void
 541 vc5_bufmgr_destroy(struct pipe_screen *pscreen)
 542 {
 543         struct vc5_screen *screen = vc5_screen(pscreen);
 544         struct vc5_bo_cache *cache = &screen->bo_cache;
 545
 546         vc5_bo_cache_free_all(cache);
 547
 548         if (dump_stats) {
 549                 fprintf(stderr, "BO stats after screen destroy:\n");
 550                 vc5_bo_dump_stats(screen);
 551         }
 552 }