src/gallium/drivers/v3d/v3d_bufmgr.c

   1 /*
   2  * Copyright © 2014-2017 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <errno.h>
  25 #include <err.h>
  26 #include <sys/mman.h>
  27 #include <fcntl.h>
  28 #include <xf86drm.h>
  29 #include <xf86drmMode.h>
  30
  31 #include "util/u_hash_table.h"
  32 #include "util/u_memory.h"
  33 #include "util/ralloc.h"
  34
  35 #include "v3d_context.h"
  36 #include "v3d_screen.h"
  37
  38 #ifdef HAVE_VALGRIND
  39 #include <valgrind.h>
  40 #include <memcheck.h>
  41 #define VG(x) x
  42 #else
  43 #define VG(x)
  44 #endif
  45
  46 static bool dump_stats = false;
  47
  48 static void
  49 v3d_bo_cache_free_all(struct v3d_bo_cache *cache);
  50
  51 static void
  52 v3d_bo_dump_stats(struct v3d_screen *screen)
  53 {
  54         struct v3d_bo_cache *cache = &screen->bo_cache;
  55
  56         uint32_t cache_count = 0;
  57         uint32_t cache_size = 0;
  58         list_for_each_entry(struct v3d_bo, bo, &cache->time_list, time_list) {
  59                 cache_count++;
  60                 cache_size += bo->size;
  61         }
  62
  63         fprintf(stderr, "  BOs allocated:   %d\n", screen->bo_count);
  64         fprintf(stderr, "  BOs size:        %dkb\n", screen->bo_size / 1024);
  65         fprintf(stderr, "  BOs cached:      %d\n", cache_count);
  66         fprintf(stderr, "  BOs cached size: %dkb\n", cache_size / 1024);
  67
  68         if (!list_is_empty(&cache->time_list)) {
  69                 struct v3d_bo *first = list_first_entry(&cache->time_list,
  70                                                         struct v3d_bo,
  71                                                         time_list);
  72                 struct v3d_bo *last = list_last_entry(&cache->time_list,
  73                                                       struct v3d_bo,
  74                                                       time_list);
  75
  76                 fprintf(stderr, "  oldest cache time: %ld\n",
  77                         (long)first->free_time);
  78                 fprintf(stderr, "  newest cache time: %ld\n",
  79                         (long)last->free_time);
  80
  81                 struct timespec time;
  82                 clock_gettime(CLOCK_MONOTONIC, &time);
  83                 fprintf(stderr, "  now:               %ld\n",
  84                         (long)time.tv_sec);
  85         }
  86 }
  87
  88 static void
  89 v3d_bo_remove_from_cache(struct v3d_bo_cache *cache, struct v3d_bo *bo)
  90 {
  91         list_del(&bo->time_list);
  92         list_del(&bo->size_list);
  93 }
  94
  95 static struct v3d_bo *
  96 v3d_bo_from_cache(struct v3d_screen *screen, uint32_t size, const char *name)
  97 {
  98         struct v3d_bo_cache *cache = &screen->bo_cache;
  99         uint32_t page_index = size / 4096 - 1;
 100
 101         if (cache->size_list_size <= page_index)
 102                 return NULL;
 103
 104         struct v3d_bo *bo = NULL;
 105         mtx_lock(&cache->lock);
 106         if (!list_is_empty(&cache->size_list[page_index])) {
 107                 bo = list_first_entry(&cache->size_list[page_index],
 108                                       struct v3d_bo, size_list);
 109
 110                 /* Check that the BO has gone idle.  If not, then we want to
 111                  * allocate something new instead, since we assume that the
 112                  * user will proceed to CPU map it and fill it with stuff.
 113                  */
 114                 if (!v3d_bo_wait(bo, 0, NULL)) {
 115                         mtx_unlock(&cache->lock);
 116                         return NULL;
 117                 }
 118
 119                 pipe_reference_init(&bo->reference, 1);
 120                 v3d_bo_remove_from_cache(cache, bo);
 121
 122                 bo->name = name;
 123         }
 124         mtx_unlock(&cache->lock);
 125         return bo;
 126 }
 127
 128 struct v3d_bo *
 129 v3d_bo_alloc(struct v3d_screen *screen, uint32_t size, const char *name)
 130 {
 131         struct v3d_bo *bo;
 132         int ret;
 133
 134         /* The CLIF dumping requires that there is no whitespace in the name.
 135          */
 136         assert(!strchr(name, ' '));
 137
 138         size = align(size, 4096);
 139
 140         bo = v3d_bo_from_cache(screen, size, name);
 141         if (bo) {
 142                 if (dump_stats) {
 143                         fprintf(stderr, "Allocated %s %dkb from cache:\n",
 144                                 name, size / 1024);
 145                         v3d_bo_dump_stats(screen);
 146                 }
 147                 return bo;
 148         }
 149
 150         bo = CALLOC_STRUCT(v3d_bo);
 151         if (!bo)
 152                 return NULL;
 153
 154         pipe_reference_init(&bo->reference, 1);
 155         bo->screen = screen;
 156         bo->size = size;
 157         bo->name = name;
 158         bo->private = true;
 159
 160  retry:
 161         ;
 162
 163         bool cleared_and_retried = false;
 164         struct drm_v3d_create_bo create = {
 165                 .size = size
 166         };
 167
 168         ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_CREATE_BO, &create);
 169         bo->handle = create.handle;
 170         bo->offset = create.offset;
 171
 172         if (ret != 0) {
 173                 if (!list_is_empty(&screen->bo_cache.time_list) &&
 174                     !cleared_and_retried) {
 175                         cleared_and_retried = true;
 176                         v3d_bo_cache_free_all(&screen->bo_cache);
 177                         goto retry;
 178                 }
 179
 180                 free(bo);
 181                 return NULL;
 182         }
 183
 184         screen->bo_count++;
 185         screen->bo_size += bo->size;
 186         if (dump_stats) {
 187                 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
 188                 v3d_bo_dump_stats(screen);
 189         }
 190
 191         return bo;
 192 }
 193
 194 void
 195 v3d_bo_last_unreference(struct v3d_bo *bo)
 196 {
 197         struct v3d_screen *screen = bo->screen;
 198
 199         struct timespec time;
 200         clock_gettime(CLOCK_MONOTONIC, &time);
 201         mtx_lock(&screen->bo_cache.lock);
 202         v3d_bo_last_unreference_locked_timed(bo, time.tv_sec);
 203         mtx_unlock(&screen->bo_cache.lock);
 204 }
 205
 206 static void
 207 v3d_bo_free(struct v3d_bo *bo)
 208 {
 209         struct v3d_screen *screen = bo->screen;
 210
 211         if (bo->map) {
 212                 if (using_v3d_simulator && bo->name &&
 213                     strcmp(bo->name, "winsys") == 0) {
 214                         free(bo->map);
 215                 } else {
 216                         munmap(bo->map, bo->size);
 217                         VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
 218                 }
 219         }
 220
 221         struct drm_gem_close c;
 222         memset(&c, 0, sizeof(c));
 223         c.handle = bo->handle;
 224         int ret = v3d_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
 225         if (ret != 0)
 226                 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
 227
 228         screen->bo_count--;
 229         screen->bo_size -= bo->size;
 230
 231         if (dump_stats) {
 232                 fprintf(stderr, "Freed %s%s%dkb:\n",
 233                         bo->name ? bo->name : "",
 234                         bo->name ? " " : "",
 235                         bo->size / 1024);
 236                 v3d_bo_dump_stats(screen);
 237         }
 238
 239         free(bo);
 240 }
 241
 242 static void
 243 free_stale_bos(struct v3d_screen *screen, time_t time)
 244 {
 245         struct v3d_bo_cache *cache = &screen->bo_cache;
 246         bool freed_any = false;
 247
 248         list_for_each_entry_safe(struct v3d_bo, bo, &cache->time_list,
 249                                  time_list) {
 250                 /* If it's more than a second old, free it. */
 251                 if (time - bo->free_time > 2) {
 252                         if (dump_stats && !freed_any) {
 253                                 fprintf(stderr, "Freeing stale BOs:\n");
 254                                 v3d_bo_dump_stats(screen);
 255                                 freed_any = true;
 256                         }
 257                         v3d_bo_remove_from_cache(cache, bo);
 258                         v3d_bo_free(bo);
 259                 } else {
 260                         break;
 261                 }
 262         }
 263
 264         if (dump_stats && freed_any) {
 265                 fprintf(stderr, "Freed stale BOs:\n");
 266                 v3d_bo_dump_stats(screen);
 267         }
 268 }
 269
 270 static void
 271 v3d_bo_cache_free_all(struct v3d_bo_cache *cache)
 272 {
 273         mtx_lock(&cache->lock);
 274         list_for_each_entry_safe(struct v3d_bo, bo, &cache->time_list,
 275                                  time_list) {
 276                 v3d_bo_remove_from_cache(cache, bo);
 277                 v3d_bo_free(bo);
 278         }
 279         mtx_unlock(&cache->lock);
 280 }
 281
 282 void
 283 v3d_bo_last_unreference_locked_timed(struct v3d_bo *bo, time_t time)
 284 {
 285         struct v3d_screen *screen = bo->screen;
 286         struct v3d_bo_cache *cache = &screen->bo_cache;
 287         uint32_t page_index = bo->size / 4096 - 1;
 288
 289         if (!bo->private) {
 290                 v3d_bo_free(bo);
 291                 return;
 292         }
 293
 294         if (cache->size_list_size <= page_index) {
 295                 struct list_head *new_list =
 296                         ralloc_array(screen, struct list_head, page_index + 1);
 297
 298                 /* Move old list contents over (since the array has moved, and
 299                  * therefore the pointers to the list heads have to change).
 300                  */
 301                 for (int i = 0; i < cache->size_list_size; i++) {
 302                         struct list_head *old_head = &cache->size_list[i];
 303                         if (list_is_empty(old_head))
 304                                 list_inithead(&new_list[i]);
 305                         else {
 306                                 new_list[i].next = old_head->next;
 307                                 new_list[i].prev = old_head->prev;
 308                                 new_list[i].next->prev = &new_list[i];
 309                                 new_list[i].prev->next = &new_list[i];
 310                         }
 311                 }
 312                 for (int i = cache->size_list_size; i < page_index + 1; i++)
 313                         list_inithead(&new_list[i]);
 314
 315                 cache->size_list = new_list;
 316                 cache->size_list_size = page_index + 1;
 317         }
 318
 319         bo->free_time = time;
 320         list_addtail(&bo->size_list, &cache->size_list[page_index]);
 321         list_addtail(&bo->time_list, &cache->time_list);
 322         if (dump_stats) {
 323                 fprintf(stderr, "Freed %s %dkb to cache:\n",
 324                         bo->name, bo->size / 1024);
 325                 v3d_bo_dump_stats(screen);
 326         }
 327         bo->name = NULL;
 328
 329         free_stale_bos(screen, time);
 330 }
 331
 332 static struct v3d_bo *
 333 v3d_bo_open_handle(struct v3d_screen *screen,
 334                    uint32_t handle, uint32_t size)
 335 {
 336         struct v3d_bo *bo;
 337
 338         assert(size);
 339
 340         mtx_lock(&screen->bo_handles_mutex);
 341
 342         bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
 343         if (bo) {
 344                 pipe_reference(NULL, &bo->reference);
 345                 goto done;
 346         }
 347
 348         bo = CALLOC_STRUCT(v3d_bo);
 349         pipe_reference_init(&bo->reference, 1);
 350         bo->screen = screen;
 351         bo->handle = handle;
 352         bo->size = size;
 353         bo->name = "winsys";
 354         bo->private = false;
 355
 356 #ifdef USE_V3D_SIMULATOR
 357         v3d_simulator_open_from_handle(screen->fd, bo->handle, bo->size);
 358         bo->map = malloc(bo->size);
 359 #endif
 360
 361         struct drm_v3d_get_bo_offset get = {
 362                 .handle = handle,
 363         };
 364         int ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_GET_BO_OFFSET, &get);
 365         if (ret) {
 366                 fprintf(stderr, "Failed to get BO offset: %s\n",
 367                         strerror(errno));
 368                 free(bo->map);
 369                 free(bo);
 370                 bo = NULL;
 371                 goto done;
 372         }
 373         bo->offset = get.offset;
 374         assert(bo->offset != 0);
 375
 376         _mesa_hash_table_insert(screen->bo_handles, (void *)(uintptr_t)handle, bo);
 377
 378         screen->bo_count++;
 379         screen->bo_size += bo->size;
 380
 381 done:
 382         mtx_unlock(&screen->bo_handles_mutex);
 383         return bo;
 384 }
 385
 386 struct v3d_bo *
 387 v3d_bo_open_name(struct v3d_screen *screen, uint32_t name)
 388 {
 389         struct drm_gem_open o = {
 390                 .name = name
 391         };
 392         int ret = v3d_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
 393         if (ret) {
 394                 fprintf(stderr, "Failed to open bo %d: %s\n",
 395                         name, strerror(errno));
 396                 return NULL;
 397         }
 398
 399         return v3d_bo_open_handle(screen, o.handle, o.size);
 400 }
 401
 402 struct v3d_bo *
 403 v3d_bo_open_dmabuf(struct v3d_screen *screen, int fd)
 404 {
 405         uint32_t handle;
 406         int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
 407         int size;
 408         if (ret) {
 409                 fprintf(stderr, "Failed to get v3d handle for dmabuf %d\n", fd);
 410                 return NULL;
 411         }
 412
 413         /* Determine the size of the bo we were handed. */
 414         size = lseek(fd, 0, SEEK_END);
 415         if (size == -1) {
 416                 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
 417                 return NULL;
 418         }
 419
 420         return v3d_bo_open_handle(screen, handle, size);
 421 }
 422
 423 int
 424 v3d_bo_get_dmabuf(struct v3d_bo *bo)
 425 {
 426         int fd;
 427         int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
 428                                      O_CLOEXEC, &fd);
 429         if (ret != 0) {
 430                 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
 431                         bo->handle);
 432                 return -1;
 433         }
 434
 435         mtx_lock(&bo->screen->bo_handles_mutex);
 436         bo->private = false;
 437         _mesa_hash_table_insert(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
 438         mtx_unlock(&bo->screen->bo_handles_mutex);
 439
 440         return fd;
 441 }
 442
 443 bool
 444 v3d_bo_flink(struct v3d_bo *bo, uint32_t *name)
 445 {
 446         struct drm_gem_flink flink = {
 447                 .handle = bo->handle,
 448         };
 449         int ret = v3d_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
 450         if (ret) {
 451                 fprintf(stderr, "Failed to flink bo %d: %s\n",
 452                         bo->handle, strerror(errno));
 453                 free(bo);
 454                 return false;
 455         }
 456
 457         bo->private = false;
 458         *name = flink.name;
 459
 460         return true;
 461 }
 462
 463 static int v3d_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
 464 {
 465         struct drm_v3d_wait_bo wait = {
 466                 .handle = handle,
 467                 .timeout_ns = timeout_ns,
 468         };
 469         int ret = v3d_ioctl(fd, DRM_IOCTL_V3D_WAIT_BO, &wait);
 470         if (ret == -1)
 471                 return -errno;
 472         else
 473                 return 0;
 474
 475 }
 476
 477 bool
 478 v3d_bo_wait(struct v3d_bo *bo, uint64_t timeout_ns, const char *reason)
 479 {
 480         struct v3d_screen *screen = bo->screen;
 481
 482         if (unlikely(V3D_DEBUG & V3D_DEBUG_PERF) && timeout_ns && reason) {
 483                 if (v3d_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
 484                         fprintf(stderr, "Blocking on %s BO for %s\n",
 485                                 bo->name, reason);
 486                 }
 487         }
 488
 489         int ret = v3d_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
 490         if (ret) {
 491                 if (ret != -ETIME) {
 492                         fprintf(stderr, "wait failed: %d\n", ret);
 493                         abort();
 494                 }
 495
 496                 return false;
 497         }
 498
 499         return true;
 500 }
 501
 502 void *
 503 v3d_bo_map_unsynchronized(struct v3d_bo *bo)
 504 {
 505         uint64_t offset;
 506         int ret;
 507
 508         if (bo->map)
 509                 return bo->map;
 510
 511         struct drm_v3d_mmap_bo map;
 512         memset(&map, 0, sizeof(map));
 513         map.handle = bo->handle;
 514         ret = v3d_ioctl(bo->screen->fd, DRM_IOCTL_V3D_MMAP_BO, &map);
 515         offset = map.offset;
 516         if (ret != 0) {
 517                 fprintf(stderr, "map ioctl failure\n");
 518                 abort();
 519         }
 520
 521         bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
 522                        bo->screen->fd, offset);
 523         if (bo->map == MAP_FAILED) {
 524                 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
 525                         bo->handle, (long long)offset, bo->size);
 526                 abort();
 527         }
 528         VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false));
 529
 530         return bo->map;
 531 }
 532
 533 void *
 534 v3d_bo_map(struct v3d_bo *bo)
 535 {
 536         void *map = v3d_bo_map_unsynchronized(bo);
 537
 538         bool ok = v3d_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
 539         if (!ok) {
 540                 fprintf(stderr, "BO wait for map failed\n");
 541                 abort();
 542         }
 543
 544         return map;
 545 }
 546
 547 void
 548 v3d_bufmgr_destroy(struct pipe_screen *pscreen)
 549 {
 550         struct v3d_screen *screen = v3d_screen(pscreen);
 551         struct v3d_bo_cache *cache = &screen->bo_cache;
 552
 553         v3d_bo_cache_free_all(cache);
 554
 555         if (dump_stats) {
 556                 fprintf(stderr, "BO stats after screen destroy:\n");
 557                 v3d_bo_dump_stats(screen);
 558         }
 559 }