src/gallium/drivers/v3d/v3d_bufmgr.c

   1 /*
   2  * Copyright © 2014-2017 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <errno.h>
  25 #include <err.h>
  26 #include <sys/mman.h>
  27 #include <fcntl.h>
  28 #include <xf86drm.h>
  29 #include <xf86drmMode.h>
  30
  31 #include "util/u_hash_table.h"
  32 #include "util/u_memory.h"
  33 #include "util/ralloc.h"
  34
  35 #include "v3d_context.h"
  36 #include "v3d_screen.h"
  37
  38 static bool dump_stats = false;
  39
  40 static void
  41 v3d_bo_cache_free_all(struct v3d_bo_cache *cache);
  42
  43 static void
  44 v3d_bo_dump_stats(struct v3d_screen *screen)
  45 {
  46         struct v3d_bo_cache *cache = &screen->bo_cache;
  47
  48         uint32_t cache_count = 0;
  49         uint32_t cache_size = 0;
  50         list_for_each_entry(struct v3d_bo, bo, &cache->time_list, time_list) {
  51                 cache_count++;
  52                 cache_size += bo->size;
  53         }
  54
  55         fprintf(stderr, "  BOs allocated:   %d\n", screen->bo_count);
  56         fprintf(stderr, "  BOs size:        %dkb\n", screen->bo_size / 1024);
  57         fprintf(stderr, "  BOs cached:      %d\n", cache_count);
  58         fprintf(stderr, "  BOs cached size: %dkb\n", cache_size / 1024);
  59
  60         if (!list_is_empty(&cache->time_list)) {
  61                 struct v3d_bo *first = list_first_entry(&cache->time_list,
  62                                                         struct v3d_bo,
  63                                                         time_list);
  64                 struct v3d_bo *last = list_last_entry(&cache->time_list,
  65                                                       struct v3d_bo,
  66                                                       time_list);
  67
  68                 fprintf(stderr, "  oldest cache time: %ld\n",
  69                         (long)first->free_time);
  70                 fprintf(stderr, "  newest cache time: %ld\n",
  71                         (long)last->free_time);
  72
  73                 struct timespec time;
  74                 clock_gettime(CLOCK_MONOTONIC, &time);
  75                 fprintf(stderr, "  now:               %ld\n",
  76                         (long)time.tv_sec);
  77         }
  78 }
  79
  80 static void
  81 v3d_bo_remove_from_cache(struct v3d_bo_cache *cache, struct v3d_bo *bo)
  82 {
  83         list_del(&bo->time_list);
  84         list_del(&bo->size_list);
  85 }
  86
  87 static struct v3d_bo *
  88 v3d_bo_from_cache(struct v3d_screen *screen, uint32_t size, const char *name)
  89 {
  90         struct v3d_bo_cache *cache = &screen->bo_cache;
  91         uint32_t page_index = size / 4096 - 1;
  92
  93         if (cache->size_list_size <= page_index)
  94                 return NULL;
  95
  96         struct v3d_bo *bo = NULL;
  97         mtx_lock(&cache->lock);
  98         if (!list_is_empty(&cache->size_list[page_index])) {
  99                 bo = list_first_entry(&cache->size_list[page_index],
 100                                       struct v3d_bo, size_list);
 101
 102                 /* Check that the BO has gone idle.  If not, then we want to
 103                  * allocate something new instead, since we assume that the
 104                  * user will proceed to CPU map it and fill it with stuff.
 105                  */
 106                 if (!v3d_bo_wait(bo, 0, NULL)) {
 107                         mtx_unlock(&cache->lock);
 108                         return NULL;
 109                 }
 110
 111                 pipe_reference_init(&bo->reference, 1);
 112                 v3d_bo_remove_from_cache(cache, bo);
 113
 114                 bo->name = name;
 115         }
 116         mtx_unlock(&cache->lock);
 117         return bo;
 118 }
 119
 120 struct v3d_bo *
 121 v3d_bo_alloc(struct v3d_screen *screen, uint32_t size, const char *name)
 122 {
 123         struct v3d_bo *bo;
 124         int ret;
 125
 126         /* The CLIF dumping requires that there is no whitespace in the name.
 127          */
 128         assert(!strchr(name, ' '));
 129
 130         size = align(size, 4096);
 131
 132         bo = v3d_bo_from_cache(screen, size, name);
 133         if (bo) {
 134                 if (dump_stats) {
 135                         fprintf(stderr, "Allocated %s %dkb from cache:\n",
 136                                 name, size / 1024);
 137                         v3d_bo_dump_stats(screen);
 138                 }
 139                 return bo;
 140         }
 141
 142         bo = CALLOC_STRUCT(v3d_bo);
 143         if (!bo)
 144                 return NULL;
 145
 146         pipe_reference_init(&bo->reference, 1);
 147         bo->screen = screen;
 148         bo->size = size;
 149         bo->name = name;
 150         bo->private = true;
 151
 152  retry:
 153         ;
 154
 155         bool cleared_and_retried = false;
 156         struct drm_v3d_create_bo create = {
 157                 .size = size
 158         };
 159
 160         ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_CREATE_BO, &create);
 161         bo->handle = create.handle;
 162         bo->offset = create.offset;
 163
 164         if (ret != 0) {
 165                 if (!list_is_empty(&screen->bo_cache.time_list) &&
 166                     !cleared_and_retried) {
 167                         cleared_and_retried = true;
 168                         v3d_bo_cache_free_all(&screen->bo_cache);
 169                         goto retry;
 170                 }
 171
 172                 free(bo);
 173                 return NULL;
 174         }
 175
 176         screen->bo_count++;
 177         screen->bo_size += bo->size;
 178         if (dump_stats) {
 179                 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
 180                 v3d_bo_dump_stats(screen);
 181         }
 182
 183         return bo;
 184 }
 185
 186 void
 187 v3d_bo_last_unreference(struct v3d_bo *bo)
 188 {
 189         struct v3d_screen *screen = bo->screen;
 190
 191         struct timespec time;
 192         clock_gettime(CLOCK_MONOTONIC, &time);
 193         mtx_lock(&screen->bo_cache.lock);
 194         v3d_bo_last_unreference_locked_timed(bo, time.tv_sec);
 195         mtx_unlock(&screen->bo_cache.lock);
 196 }
 197
 198 static void
 199 v3d_bo_free(struct v3d_bo *bo)
 200 {
 201         struct v3d_screen *screen = bo->screen;
 202
 203         if (bo->map) {
 204                 if (using_v3d_simulator && bo->name &&
 205                     strcmp(bo->name, "winsys") == 0) {
 206                         free(bo->map);
 207                 } else {
 208                         munmap(bo->map, bo->size);
 209                         VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
 210                 }
 211         }
 212
 213         struct drm_gem_close c;
 214         memset(&c, 0, sizeof(c));
 215         c.handle = bo->handle;
 216         int ret = v3d_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
 217         if (ret != 0)
 218                 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
 219
 220         screen->bo_count--;
 221         screen->bo_size -= bo->size;
 222
 223         if (dump_stats) {
 224                 fprintf(stderr, "Freed %s%s%dkb:\n",
 225                         bo->name ? bo->name : "",
 226                         bo->name ? " " : "",
 227                         bo->size / 1024);
 228                 v3d_bo_dump_stats(screen);
 229         }
 230
 231         free(bo);
 232 }
 233
 234 static void
 235 free_stale_bos(struct v3d_screen *screen, time_t time)
 236 {
 237         struct v3d_bo_cache *cache = &screen->bo_cache;
 238         bool freed_any = false;
 239
 240         list_for_each_entry_safe(struct v3d_bo, bo, &cache->time_list,
 241                                  time_list) {
 242                 /* If it's more than a second old, free it. */
 243                 if (time - bo->free_time > 2) {
 244                         if (dump_stats && !freed_any) {
 245                                 fprintf(stderr, "Freeing stale BOs:\n");
 246                                 v3d_bo_dump_stats(screen);
 247                                 freed_any = true;
 248                         }
 249                         v3d_bo_remove_from_cache(cache, bo);
 250                         v3d_bo_free(bo);
 251                 } else {
 252                         break;
 253                 }
 254         }
 255
 256         if (dump_stats && freed_any) {
 257                 fprintf(stderr, "Freed stale BOs:\n");
 258                 v3d_bo_dump_stats(screen);
 259         }
 260 }
 261
 262 static void
 263 v3d_bo_cache_free_all(struct v3d_bo_cache *cache)
 264 {
 265         mtx_lock(&cache->lock);
 266         list_for_each_entry_safe(struct v3d_bo, bo, &cache->time_list,
 267                                  time_list) {
 268                 v3d_bo_remove_from_cache(cache, bo);
 269                 v3d_bo_free(bo);
 270         }
 271         mtx_unlock(&cache->lock);
 272 }
 273
 274 void
 275 v3d_bo_last_unreference_locked_timed(struct v3d_bo *bo, time_t time)
 276 {
 277         struct v3d_screen *screen = bo->screen;
 278         struct v3d_bo_cache *cache = &screen->bo_cache;
 279         uint32_t page_index = bo->size / 4096 - 1;
 280
 281         if (!bo->private) {
 282                 v3d_bo_free(bo);
 283                 return;
 284         }
 285
 286         if (cache->size_list_size <= page_index) {
 287                 struct list_head *new_list =
 288                         ralloc_array(screen, struct list_head, page_index + 1);
 289
 290                 /* Move old list contents over (since the array has moved, and
 291                  * therefore the pointers to the list heads have to change).
 292                  */
 293                 for (int i = 0; i < cache->size_list_size; i++) {
 294                         struct list_head *old_head = &cache->size_list[i];
 295                         if (list_is_empty(old_head))
 296                                 list_inithead(&new_list[i]);
 297                         else {
 298                                 new_list[i].next = old_head->next;
 299                                 new_list[i].prev = old_head->prev;
 300                                 new_list[i].next->prev = &new_list[i];
 301                                 new_list[i].prev->next = &new_list[i];
 302                         }
 303                 }
 304                 for (int i = cache->size_list_size; i < page_index + 1; i++)
 305                         list_inithead(&new_list[i]);
 306
 307                 cache->size_list = new_list;
 308                 cache->size_list_size = page_index + 1;
 309         }
 310
 311         bo->free_time = time;
 312         list_addtail(&bo->size_list, &cache->size_list[page_index]);
 313         list_addtail(&bo->time_list, &cache->time_list);
 314         if (dump_stats) {
 315                 fprintf(stderr, "Freed %s %dkb to cache:\n",
 316                         bo->name, bo->size / 1024);
 317                 v3d_bo_dump_stats(screen);
 318         }
 319         bo->name = NULL;
 320
 321         free_stale_bos(screen, time);
 322 }
 323
 324 static struct v3d_bo *
 325 v3d_bo_open_handle(struct v3d_screen *screen,
 326                    uint32_t handle, uint32_t size)
 327 {
 328         struct v3d_bo *bo;
 329
 330         assert(size);
 331
 332         mtx_lock(&screen->bo_handles_mutex);
 333
 334         bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
 335         if (bo) {
 336                 pipe_reference(NULL, &bo->reference);
 337                 goto done;
 338         }
 339
 340         bo = CALLOC_STRUCT(v3d_bo);
 341         pipe_reference_init(&bo->reference, 1);
 342         bo->screen = screen;
 343         bo->handle = handle;
 344         bo->size = size;
 345         bo->name = "winsys";
 346         bo->private = false;
 347
 348 #ifdef USE_V3D_SIMULATOR
 349         v3d_simulator_open_from_handle(screen->fd, bo->handle, bo->size);
 350         bo->map = malloc(bo->size);
 351 #endif
 352
 353         struct drm_v3d_get_bo_offset get = {
 354                 .handle = handle,
 355         };
 356         int ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_GET_BO_OFFSET, &get);
 357         if (ret) {
 358                 fprintf(stderr, "Failed to get BO offset: %s\n",
 359                         strerror(errno));
 360                 free(bo->map);
 361                 free(bo);
 362                 bo = NULL;
 363                 goto done;
 364         }
 365         bo->offset = get.offset;
 366         assert(bo->offset != 0);
 367
 368         _mesa_hash_table_insert(screen->bo_handles, (void *)(uintptr_t)handle, bo);
 369
 370         screen->bo_count++;
 371         screen->bo_size += bo->size;
 372
 373 done:
 374         mtx_unlock(&screen->bo_handles_mutex);
 375         return bo;
 376 }
 377
 378 struct v3d_bo *
 379 v3d_bo_open_name(struct v3d_screen *screen, uint32_t name)
 380 {
 381         struct drm_gem_open o = {
 382                 .name = name
 383         };
 384         int ret = v3d_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
 385         if (ret) {
 386                 fprintf(stderr, "Failed to open bo %d: %s\n",
 387                         name, strerror(errno));
 388                 return NULL;
 389         }
 390
 391         return v3d_bo_open_handle(screen, o.handle, o.size);
 392 }
 393
 394 struct v3d_bo *
 395 v3d_bo_open_dmabuf(struct v3d_screen *screen, int fd)
 396 {
 397         uint32_t handle;
 398         int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
 399         int size;
 400         if (ret) {
 401                 fprintf(stderr, "Failed to get v3d handle for dmabuf %d\n", fd);
 402                 return NULL;
 403         }
 404
 405         /* Determine the size of the bo we were handed. */
 406         size = lseek(fd, 0, SEEK_END);
 407         if (size == -1) {
 408                 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
 409                 return NULL;
 410         }
 411
 412         return v3d_bo_open_handle(screen, handle, size);
 413 }
 414
 415 int
 416 v3d_bo_get_dmabuf(struct v3d_bo *bo)
 417 {
 418         int fd;
 419         int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
 420                                      O_CLOEXEC, &fd);
 421         if (ret != 0) {
 422                 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
 423                         bo->handle);
 424                 return -1;
 425         }
 426
 427         mtx_lock(&bo->screen->bo_handles_mutex);
 428         bo->private = false;
 429         _mesa_hash_table_insert(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
 430         mtx_unlock(&bo->screen->bo_handles_mutex);
 431
 432         return fd;
 433 }
 434
 435 bool
 436 v3d_bo_flink(struct v3d_bo *bo, uint32_t *name)
 437 {
 438         struct drm_gem_flink flink = {
 439                 .handle = bo->handle,
 440         };
 441         int ret = v3d_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
 442         if (ret) {
 443                 fprintf(stderr, "Failed to flink bo %d: %s\n",
 444                         bo->handle, strerror(errno));
 445                 free(bo);
 446                 return false;
 447         }
 448
 449         bo->private = false;
 450         *name = flink.name;
 451
 452         return true;
 453 }
 454
 455 static int v3d_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
 456 {
 457         struct drm_v3d_wait_bo wait = {
 458                 .handle = handle,
 459                 .timeout_ns = timeout_ns,
 460         };
 461         int ret = v3d_ioctl(fd, DRM_IOCTL_V3D_WAIT_BO, &wait);
 462         if (ret == -1)
 463                 return -errno;
 464         else
 465                 return 0;
 466
 467 }
 468
 469 bool
 470 v3d_bo_wait(struct v3d_bo *bo, uint64_t timeout_ns, const char *reason)
 471 {
 472         struct v3d_screen *screen = bo->screen;
 473
 474         if (unlikely(V3D_DEBUG & V3D_DEBUG_PERF) && timeout_ns && reason) {
 475                 if (v3d_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
 476                         fprintf(stderr, "Blocking on %s BO for %s\n",
 477                                 bo->name, reason);
 478                 }
 479         }
 480
 481         int ret = v3d_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
 482         if (ret) {
 483                 if (ret != -ETIME) {
 484                         fprintf(stderr, "wait failed: %d\n", ret);
 485                         abort();
 486                 }
 487
 488                 return false;
 489         }
 490
 491         return true;
 492 }
 493
 494 void *
 495 v3d_bo_map_unsynchronized(struct v3d_bo *bo)
 496 {
 497         uint64_t offset;
 498         int ret;
 499
 500         if (bo->map)
 501                 return bo->map;
 502
 503         struct drm_v3d_mmap_bo map;
 504         memset(&map, 0, sizeof(map));
 505         map.handle = bo->handle;
 506         ret = v3d_ioctl(bo->screen->fd, DRM_IOCTL_V3D_MMAP_BO, &map);
 507         offset = map.offset;
 508         if (ret != 0) {
 509                 fprintf(stderr, "map ioctl failure\n");
 510                 abort();
 511         }
 512
 513         bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
 514                        bo->screen->fd, offset);
 515         if (bo->map == MAP_FAILED) {
 516                 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
 517                         bo->handle, (long long)offset, bo->size);
 518                 abort();
 519         }
 520         VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false));
 521
 522         return bo->map;
 523 }
 524
 525 void *
 526 v3d_bo_map(struct v3d_bo *bo)
 527 {
 528         void *map = v3d_bo_map_unsynchronized(bo);
 529
 530         bool ok = v3d_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
 531         if (!ok) {
 532                 fprintf(stderr, "BO wait for map failed\n");
 533                 abort();
 534         }
 535
 536         return map;
 537 }
 538
 539 void
 540 v3d_bufmgr_destroy(struct pipe_screen *pscreen)
 541 {
 542         struct v3d_screen *screen = v3d_screen(pscreen);
 543         struct v3d_bo_cache *cache = &screen->bo_cache;
 544
 545         v3d_bo_cache_free_all(cache);
 546
 547         if (dump_stats) {
 548                 fprintf(stderr, "BO stats after screen destroy:\n");
 549                 v3d_bo_dump_stats(screen);
 550         }
 551 }