src/gallium/drivers/v3d/v3d_bufmgr.c

   1 /*
   2  * Copyright © 2014-2017 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <errno.h>
  25 #include <err.h>
  26 #include <sys/mman.h>
  27 #include <fcntl.h>
  28 #include <xf86drm.h>
  29 #include <xf86drmMode.h>
  30
  31 #include "util/u_hash_table.h"
  32 #include "util/u_memory.h"
  33 #include "util/ralloc.h"
  34
  35 #include "v3d_context.h"
  36 #include "v3d_screen.h"
  37
  38 #ifdef HAVE_VALGRIND
  39 #include <valgrind.h>
  40 #include <memcheck.h>
  41 #define VG(x) x
  42 #else
  43 #define VG(x)
  44 #endif
  45
  46 static bool dump_stats = false;
  47
  48 static void
  49 v3d_bo_cache_free_all(struct v3d_bo_cache *cache);
  50
  51 static void
  52 v3d_bo_dump_stats(struct v3d_screen *screen)
  53 {
  54         struct v3d_bo_cache *cache = &screen->bo_cache;
  55
  56         uint32_t cache_count = 0;
  57         uint32_t cache_size = 0;
  58         list_for_each_entry(struct v3d_bo, bo, &cache->time_list, time_list) {
  59                 cache_count++;
  60                 cache_size += bo->size;
  61         }
  62
  63         fprintf(stderr, "  BOs allocated:   %d\n", screen->bo_count);
  64         fprintf(stderr, "  BOs size:        %dkb\n", screen->bo_size / 1024);
  65         fprintf(stderr, "  BOs cached:      %d\n", cache_count);
  66         fprintf(stderr, "  BOs cached size: %dkb\n", cache_size / 1024);
  67
  68         if (!list_empty(&cache->time_list)) {
  69                 struct v3d_bo *first = LIST_ENTRY(struct v3d_bo,
  70                                                   cache->time_list.next,
  71                                                   time_list);
  72                 struct v3d_bo *last = LIST_ENTRY(struct v3d_bo,
  73                                                   cache->time_list.prev,
  74                                                   time_list);
  75
  76                 fprintf(stderr, "  oldest cache time: %ld\n",
  77                         (long)first->free_time);
  78                 fprintf(stderr, "  newest cache time: %ld\n",
  79                         (long)last->free_time);
  80
  81                 struct timespec time;
  82                 clock_gettime(CLOCK_MONOTONIC, &time);
  83                 fprintf(stderr, "  now:               %ld\n",
  84                         time.tv_sec);
  85         }
  86 }
  87
  88 static void
  89 v3d_bo_remove_from_cache(struct v3d_bo_cache *cache, struct v3d_bo *bo)
  90 {
  91         list_del(&bo->time_list);
  92         list_del(&bo->size_list);
  93 }
  94
  95 static struct v3d_bo *
  96 v3d_bo_from_cache(struct v3d_screen *screen, uint32_t size, const char *name)
  97 {
  98         struct v3d_bo_cache *cache = &screen->bo_cache;
  99         uint32_t page_index = size / 4096 - 1;
 100
 101         if (cache->size_list_size <= page_index)
 102                 return NULL;
 103
 104         struct v3d_bo *bo = NULL;
 105         mtx_lock(&cache->lock);
 106         if (!list_empty(&cache->size_list[page_index])) {
 107                 bo = LIST_ENTRY(struct v3d_bo, cache->size_list[page_index].next,
 108                                 size_list);
 109
 110                 /* Check that the BO has gone idle.  If not, then we want to
 111                  * allocate something new instead, since we assume that the
 112                  * user will proceed to CPU map it and fill it with stuff.
 113                  */
 114                 if (!v3d_bo_wait(bo, 0, NULL)) {
 115                         mtx_unlock(&cache->lock);
 116                         return NULL;
 117                 }
 118
 119                 pipe_reference_init(&bo->reference, 1);
 120                 v3d_bo_remove_from_cache(cache, bo);
 121
 122                 bo->name = name;
 123         }
 124         mtx_unlock(&cache->lock);
 125         return bo;
 126 }
 127
 128 struct v3d_bo *
 129 v3d_bo_alloc(struct v3d_screen *screen, uint32_t size, const char *name)
 130 {
 131         struct v3d_bo *bo;
 132         int ret;
 133
 134         size = align(size, 4096);
 135
 136         bo = v3d_bo_from_cache(screen, size, name);
 137         if (bo) {
 138                 if (dump_stats) {
 139                         fprintf(stderr, "Allocated %s %dkb from cache:\n",
 140                                 name, size / 1024);
 141                         v3d_bo_dump_stats(screen);
 142                 }
 143                 return bo;
 144         }
 145
 146         bo = CALLOC_STRUCT(v3d_bo);
 147         if (!bo)
 148                 return NULL;
 149
 150         pipe_reference_init(&bo->reference, 1);
 151         bo->screen = screen;
 152         bo->size = size;
 153         bo->name = name;
 154         bo->private = true;
 155
 156  retry:
 157         ;
 158
 159         bool cleared_and_retried = false;
 160         struct drm_v3d_create_bo create = {
 161                 .size = size
 162         };
 163
 164         ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_CREATE_BO, &create);
 165         bo->handle = create.handle;
 166         bo->offset = create.offset;
 167
 168         if (ret != 0) {
 169                 if (!list_empty(&screen->bo_cache.time_list) &&
 170                     !cleared_and_retried) {
 171                         cleared_and_retried = true;
 172                         v3d_bo_cache_free_all(&screen->bo_cache);
 173                         goto retry;
 174                 }
 175
 176                 free(bo);
 177                 return NULL;
 178         }
 179
 180         screen->bo_count++;
 181         screen->bo_size += bo->size;
 182         if (dump_stats) {
 183                 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
 184                 v3d_bo_dump_stats(screen);
 185         }
 186
 187         return bo;
 188 }
 189
 190 void
 191 v3d_bo_last_unreference(struct v3d_bo *bo)
 192 {
 193         struct v3d_screen *screen = bo->screen;
 194
 195         struct timespec time;
 196         clock_gettime(CLOCK_MONOTONIC, &time);
 197         mtx_lock(&screen->bo_cache.lock);
 198         v3d_bo_last_unreference_locked_timed(bo, time.tv_sec);
 199         mtx_unlock(&screen->bo_cache.lock);
 200 }
 201
 202 static void
 203 v3d_bo_free(struct v3d_bo *bo)
 204 {
 205         struct v3d_screen *screen = bo->screen;
 206
 207         if (bo->map) {
 208                 if (using_v3d_simulator && bo->name &&
 209                     strcmp(bo->name, "winsys") == 0) {
 210                         free(bo->map);
 211                 } else {
 212                         munmap(bo->map, bo->size);
 213                         VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
 214                 }
 215         }
 216
 217         struct drm_gem_close c;
 218         memset(&c, 0, sizeof(c));
 219         c.handle = bo->handle;
 220         int ret = v3d_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
 221         if (ret != 0)
 222                 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
 223
 224         screen->bo_count--;
 225         screen->bo_size -= bo->size;
 226
 227         if (dump_stats) {
 228                 fprintf(stderr, "Freed %s%s%dkb:\n",
 229                         bo->name ? bo->name : "",
 230                         bo->name ? " " : "",
 231                         bo->size / 1024);
 232                 v3d_bo_dump_stats(screen);
 233         }
 234
 235         free(bo);
 236 }
 237
 238 static void
 239 free_stale_bos(struct v3d_screen *screen, time_t time)
 240 {
 241         struct v3d_bo_cache *cache = &screen->bo_cache;
 242         bool freed_any = false;
 243
 244         list_for_each_entry_safe(struct v3d_bo, bo, &cache->time_list,
 245                                  time_list) {
 246                 /* If it's more than a second old, free it. */
 247                 if (time - bo->free_time > 2) {
 248                         if (dump_stats && !freed_any) {
 249                                 fprintf(stderr, "Freeing stale BOs:\n");
 250                                 v3d_bo_dump_stats(screen);
 251                                 freed_any = true;
 252                         }
 253                         v3d_bo_remove_from_cache(cache, bo);
 254                         v3d_bo_free(bo);
 255                 } else {
 256                         break;
 257                 }
 258         }
 259
 260         if (dump_stats && freed_any) {
 261                 fprintf(stderr, "Freed stale BOs:\n");
 262                 v3d_bo_dump_stats(screen);
 263         }
 264 }
 265
 266 static void
 267 v3d_bo_cache_free_all(struct v3d_bo_cache *cache)
 268 {
 269         mtx_lock(&cache->lock);
 270         list_for_each_entry_safe(struct v3d_bo, bo, &cache->time_list,
 271                                  time_list) {
 272                 v3d_bo_remove_from_cache(cache, bo);
 273                 v3d_bo_free(bo);
 274         }
 275         mtx_unlock(&cache->lock);
 276 }
 277
 278 void
 279 v3d_bo_last_unreference_locked_timed(struct v3d_bo *bo, time_t time)
 280 {
 281         struct v3d_screen *screen = bo->screen;
 282         struct v3d_bo_cache *cache = &screen->bo_cache;
 283         uint32_t page_index = bo->size / 4096 - 1;
 284
 285         if (!bo->private) {
 286                 v3d_bo_free(bo);
 287                 return;
 288         }
 289
 290         if (cache->size_list_size <= page_index) {
 291                 struct list_head *new_list =
 292                         ralloc_array(screen, struct list_head, page_index + 1);
 293
 294                 /* Move old list contents over (since the array has moved, and
 295                  * therefore the pointers to the list heads have to change).
 296                  */
 297                 for (int i = 0; i < cache->size_list_size; i++) {
 298                         struct list_head *old_head = &cache->size_list[i];
 299                         if (list_empty(old_head))
 300                                 list_inithead(&new_list[i]);
 301                         else {
 302                                 new_list[i].next = old_head->next;
 303                                 new_list[i].prev = old_head->prev;
 304                                 new_list[i].next->prev = &new_list[i];
 305                                 new_list[i].prev->next = &new_list[i];
 306                         }
 307                 }
 308                 for (int i = cache->size_list_size; i < page_index + 1; i++)
 309                         list_inithead(&new_list[i]);
 310
 311                 cache->size_list = new_list;
 312                 cache->size_list_size = page_index + 1;
 313         }
 314
 315         bo->free_time = time;
 316         list_addtail(&bo->size_list, &cache->size_list[page_index]);
 317         list_addtail(&bo->time_list, &cache->time_list);
 318         if (dump_stats) {
 319                 fprintf(stderr, "Freed %s %dkb to cache:\n",
 320                         bo->name, bo->size / 1024);
 321                 v3d_bo_dump_stats(screen);
 322         }
 323         bo->name = NULL;
 324
 325         free_stale_bos(screen, time);
 326 }
 327
 328 static struct v3d_bo *
 329 v3d_bo_open_handle(struct v3d_screen *screen,
 330                    uint32_t winsys_stride,
 331                    uint32_t handle, uint32_t size)
 332 {
 333         struct v3d_bo *bo;
 334
 335         assert(size);
 336
 337         mtx_lock(&screen->bo_handles_mutex);
 338
 339         bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
 340         if (bo) {
 341                 pipe_reference(NULL, &bo->reference);
 342                 goto done;
 343         }
 344
 345         bo = CALLOC_STRUCT(v3d_bo);
 346         pipe_reference_init(&bo->reference, 1);
 347         bo->screen = screen;
 348         bo->handle = handle;
 349         bo->size = size;
 350         bo->name = "winsys";
 351         bo->private = false;
 352
 353 #ifdef USE_V3D_SIMULATOR
 354         v3d_simulator_open_from_handle(screen->fd, winsys_stride,
 355                                        bo->handle, bo->size);
 356         bo->map = malloc(bo->size);
 357 #endif
 358
 359         struct drm_v3d_get_bo_offset get = {
 360                 .handle = handle,
 361         };
 362         int ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_GET_BO_OFFSET, &get);
 363         if (ret) {
 364                 fprintf(stderr, "Failed to get BO offset: %s\n",
 365                         strerror(errno));
 366                 free(bo->map);
 367                 free(bo);
 368                 return NULL;
 369         }
 370         bo->offset = get.offset;
 371         assert(bo->offset != 0);
 372
 373         util_hash_table_set(screen->bo_handles, (void *)(uintptr_t)handle, bo);
 374
 375 done:
 376         mtx_unlock(&screen->bo_handles_mutex);
 377         return bo;
 378 }
 379
 380 struct v3d_bo *
 381 v3d_bo_open_name(struct v3d_screen *screen, uint32_t name,
 382                  uint32_t winsys_stride)
 383 {
 384         struct drm_gem_open o = {
 385                 .name = name
 386         };
 387         int ret = v3d_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
 388         if (ret) {
 389                 fprintf(stderr, "Failed to open bo %d: %s\n",
 390                         name, strerror(errno));
 391                 return NULL;
 392         }
 393
 394         return v3d_bo_open_handle(screen, winsys_stride, o.handle, o.size);
 395 }
 396
 397 struct v3d_bo *
 398 v3d_bo_open_dmabuf(struct v3d_screen *screen, int fd, uint32_t winsys_stride)
 399 {
 400         uint32_t handle;
 401         int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
 402         int size;
 403         if (ret) {
 404                 fprintf(stderr, "Failed to get v3d handle for dmabuf %d\n", fd);
 405                 return NULL;
 406         }
 407
 408         /* Determine the size of the bo we were handed. */
 409         size = lseek(fd, 0, SEEK_END);
 410         if (size == -1) {
 411                 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
 412                 return NULL;
 413         }
 414
 415         return v3d_bo_open_handle(screen, winsys_stride, handle, size);
 416 }
 417
 418 int
 419 v3d_bo_get_dmabuf(struct v3d_bo *bo)
 420 {
 421         int fd;
 422         int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
 423                                      O_CLOEXEC, &fd);
 424         if (ret != 0) {
 425                 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
 426                         bo->handle);
 427                 return -1;
 428         }
 429
 430         mtx_lock(&bo->screen->bo_handles_mutex);
 431         bo->private = false;
 432         util_hash_table_set(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
 433         mtx_unlock(&bo->screen->bo_handles_mutex);
 434
 435         return fd;
 436 }
 437
 438 bool
 439 v3d_bo_flink(struct v3d_bo *bo, uint32_t *name)
 440 {
 441         struct drm_gem_flink flink = {
 442                 .handle = bo->handle,
 443         };
 444         int ret = v3d_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
 445         if (ret) {
 446                 fprintf(stderr, "Failed to flink bo %d: %s\n",
 447                         bo->handle, strerror(errno));
 448                 free(bo);
 449                 return false;
 450         }
 451
 452         bo->private = false;
 453         *name = flink.name;
 454
 455         return true;
 456 }
 457
 458 static int v3d_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
 459 {
 460         struct drm_v3d_wait_bo wait = {
 461                 .handle = handle,
 462                 .timeout_ns = timeout_ns,
 463         };
 464         int ret = v3d_ioctl(fd, DRM_IOCTL_V3D_WAIT_BO, &wait);
 465         if (ret == -1)
 466                 return -errno;
 467         else
 468                 return 0;
 469
 470 }
 471
 472 bool
 473 v3d_bo_wait(struct v3d_bo *bo, uint64_t timeout_ns, const char *reason)
 474 {
 475         struct v3d_screen *screen = bo->screen;
 476
 477         if (unlikely(V3D_DEBUG & V3D_DEBUG_PERF) && timeout_ns && reason) {
 478                 if (v3d_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
 479                         fprintf(stderr, "Blocking on %s BO for %s\n",
 480                                 bo->name, reason);
 481                 }
 482         }
 483
 484         int ret = v3d_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
 485         if (ret) {
 486                 if (ret != -ETIME) {
 487                         fprintf(stderr, "wait failed: %d\n", ret);
 488                         abort();
 489                 }
 490
 491                 return false;
 492         }
 493
 494         return true;
 495 }
 496
 497 void *
 498 v3d_bo_map_unsynchronized(struct v3d_bo *bo)
 499 {
 500         uint64_t offset;
 501         int ret;
 502
 503         if (bo->map)
 504                 return bo->map;
 505
 506         struct drm_v3d_mmap_bo map;
 507         memset(&map, 0, sizeof(map));
 508         map.handle = bo->handle;
 509         ret = v3d_ioctl(bo->screen->fd, DRM_IOCTL_V3D_MMAP_BO, &map);
 510         offset = map.offset;
 511         if (ret != 0) {
 512                 fprintf(stderr, "map ioctl failure\n");
 513                 abort();
 514         }
 515
 516         bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
 517                        bo->screen->fd, offset);
 518         if (bo->map == MAP_FAILED) {
 519                 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
 520                         bo->handle, (long long)offset, bo->size);
 521                 abort();
 522         }
 523         VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false));
 524
 525         return bo->map;
 526 }
 527
 528 void *
 529 v3d_bo_map(struct v3d_bo *bo)
 530 {
 531         void *map = v3d_bo_map_unsynchronized(bo);
 532
 533         bool ok = v3d_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
 534         if (!ok) {
 535                 fprintf(stderr, "BO wait for map failed\n");
 536                 abort();
 537         }
 538
 539         return map;
 540 }
 541
 542 void
 543 v3d_bufmgr_destroy(struct pipe_screen *pscreen)
 544 {
 545         struct v3d_screen *screen = v3d_screen(pscreen);
 546         struct v3d_bo_cache *cache = &screen->bo_cache;
 547
 548         v3d_bo_cache_free_all(cache);
 549
 550         if (dump_stats) {
 551                 fprintf(stderr, "BO stats after screen destroy:\n");
 552                 v3d_bo_dump_stats(screen);
 553         }
 554 }