src/gallium/drivers/vc4/vc4_bufmgr.c

   1 /*
   2  * Copyright © 2014 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <errno.h>
  25 #include <err.h>
  26 #include <sys/mman.h>
  27 #include <fcntl.h>
  28 #include <xf86drm.h>
  29 #include <xf86drmMode.h>
  30
  31 #include "util/u_memory.h"
  32 #include "util/ralloc.h"
  33
  34 #include "vc4_context.h"
  35 #include "vc4_screen.h"
  36
  37 static bool dump_stats = false;
  38
  39 static void
  40 vc4_bo_dump_stats(struct vc4_screen *screen)
  41 {
  42         struct vc4_bo_cache *cache = &screen->bo_cache;
  43
  44         fprintf(stderr, "  BOs allocated:   %d\n", screen->bo_count);
  45         fprintf(stderr, "  BOs size:        %dkb\n", screen->bo_size / 102);
  46         fprintf(stderr, "  BOs cached:      %d\n", cache->bo_count);
  47         fprintf(stderr, "  BOs cached size: %dkb\n", cache->bo_size / 102);
  48
  49         if (!list_empty(&cache->time_list)) {
  50                 struct vc4_bo *first = LIST_ENTRY(struct vc4_bo,
  51                                                   cache->time_list.next,
  52                                                   time_list);
  53                 struct vc4_bo *last = LIST_ENTRY(struct vc4_bo,
  54                                                   cache->time_list.prev,
  55                                                   time_list);
  56
  57                 fprintf(stderr, "  oldest cache time: %ld\n",
  58                         (long)first->free_time);
  59                 fprintf(stderr, "  newest cache time: %ld\n",
  60                         (long)last->free_time);
  61
  62                 struct timespec time;
  63                 clock_gettime(CLOCK_MONOTONIC, &time);
  64                 fprintf(stderr, "  now:               %ld\n",
  65                         time.tv_sec);
  66         }
  67 }
  68
  69 static void
  70 vc4_bo_remove_from_cache(struct vc4_bo_cache *cache, struct vc4_bo *bo)
  71 {
  72         list_del(&bo->time_list);
  73         list_del(&bo->size_list);
  74         cache->bo_count--;
  75         cache->bo_size -= bo->size;
  76 }
  77
  78 static struct vc4_bo *
  79 vc4_bo_from_cache(struct vc4_screen *screen, uint32_t size, const char *name)
  80 {
  81         struct vc4_bo_cache *cache = &screen->bo_cache;
  82         uint32_t page_index = size / 4096 - 1;
  83
  84         if (cache->size_list_size <= page_index)
  85                 return NULL;
  86
  87         struct vc4_bo *bo = NULL;
  88         pipe_mutex_lock(cache->lock);
  89         if (!list_empty(&cache->size_list[page_index])) {
  90                 struct vc4_bo *bo = LIST_ENTRY(struct vc4_bo,
  91                                                cache->size_list[page_index].next,
  92                                                size_list);
  93
  94                 /* Check that the BO has gone idle.  If not, then we want to
  95                  * allocate something new instead, since we assume that the
  96                  * user will proceed to CPU map it and fill it with stuff.
  97                  */
  98                 if (!vc4_bo_wait(bo, 0)) {
  99                         pipe_mutex_unlock(cache->lock);
 100                         return NULL;
 101                 }
 102
 103                 pipe_reference_init(&bo->reference, 1);
 104                 vc4_bo_remove_from_cache(cache, bo);
 105
 106                 bo->name = name;
 107         }
 108         pipe_mutex_unlock(cache->lock);
 109         return bo;
 110 }
 111
 112 struct vc4_bo *
 113 vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name)
 114 {
 115         struct vc4_bo *bo;
 116         int ret;
 117
 118         size = align(size, 4096);
 119
 120         bo = vc4_bo_from_cache(screen, size, name);
 121         if (bo) {
 122                 if (dump_stats) {
 123                         fprintf(stderr, "Allocated %s %dkb from cache:\n",
 124                                 name, size / 1024);
 125                         vc4_bo_dump_stats(screen);
 126                 }
 127                 return bo;
 128         }
 129
 130         bo = CALLOC_STRUCT(vc4_bo);
 131         if (!bo)
 132                 return NULL;
 133
 134         pipe_reference_init(&bo->reference, 1);
 135         bo->screen = screen;
 136         bo->size = size;
 137         bo->name = name;
 138         bo->private = true;
 139
 140         if (!using_vc4_simulator) {
 141                 struct drm_vc4_create_bo create;
 142                 memset(&create, 0, sizeof(create));
 143
 144                 create.size = size;
 145
 146                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_CREATE_BO, &create);
 147                 bo->handle = create.handle;
 148         } else {
 149                 struct drm_mode_create_dumb create;
 150                 memset(&create, 0, sizeof(create));
 151
 152                 create.width = 128;
 153                 create.bpp = 8;
 154                 create.height = (size + 127) / 128;
 155
 156                 ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
 157                 bo->handle = create.handle;
 158                 assert(create.size >= size);
 159         }
 160         if (ret != 0) {
 161                 fprintf(stderr, "create ioctl failure\n");
 162                 abort();
 163         }
 164
 165         screen->bo_count++;
 166         screen->bo_size += bo->size;
 167         if (dump_stats) {
 168                 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
 169                 vc4_bo_dump_stats(screen);
 170         }
 171
 172         return bo;
 173 }
 174
 175 void
 176 vc4_bo_last_unreference(struct vc4_bo *bo)
 177 {
 178         struct vc4_screen *screen = bo->screen;
 179
 180         struct timespec time;
 181         clock_gettime(CLOCK_MONOTONIC, &time);
 182         pipe_mutex_lock(screen->bo_cache.lock);
 183         vc4_bo_last_unreference_locked_timed(bo, time.tv_sec);
 184         pipe_mutex_unlock(screen->bo_cache.lock);
 185 }
 186
 187 static void
 188 vc4_bo_free(struct vc4_bo *bo)
 189 {
 190         struct vc4_screen *screen = bo->screen;
 191
 192         if (bo->map) {
 193 #ifdef USE_VC4_SIMULATOR
 194                 if (bo->simulator_winsys_map) {
 195                         free(bo->map);
 196                         bo->map = bo->simulator_winsys_map;
 197                 }
 198 #endif
 199                 munmap(bo->map, bo->size);
 200         }
 201
 202         struct drm_gem_close c;
 203         memset(&c, 0, sizeof(c));
 204         c.handle = bo->handle;
 205         int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
 206         if (ret != 0)
 207                 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
 208
 209         screen->bo_count--;
 210         screen->bo_size -= bo->size;
 211
 212         if (dump_stats) {
 213                 fprintf(stderr, "Freed %s%s%dkb:\n",
 214                         bo->name ? bo->name : "",
 215                         bo->name ? " " : "",
 216                         bo->size / 1024);
 217                 vc4_bo_dump_stats(screen);
 218         }
 219
 220         free(bo);
 221 }
 222
 223 static void
 224 free_stale_bos(struct vc4_screen *screen, time_t time)
 225 {
 226         struct vc4_bo_cache *cache = &screen->bo_cache;
 227         bool freed_any = false;
 228
 229         list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
 230                                  time_list) {
 231                 if (dump_stats && !freed_any) {
 232                         fprintf(stderr, "Freeing stale BOs:\n");
 233                         vc4_bo_dump_stats(screen);
 234                         freed_any = true;
 235                 }
 236
 237                 /* If it's more than a second old, free it. */
 238                 if (time - bo->free_time > 2) {
 239                         vc4_bo_remove_from_cache(cache, bo);
 240                         vc4_bo_free(bo);
 241                 } else {
 242                         break;
 243                 }
 244         }
 245
 246         if (dump_stats && freed_any) {
 247                 fprintf(stderr, "Freed stale BOs:\n");
 248                 vc4_bo_dump_stats(screen);
 249         }
 250 }
 251
 252 void
 253 vc4_bo_last_unreference_locked_timed(struct vc4_bo *bo, time_t time)
 254 {
 255         struct vc4_screen *screen = bo->screen;
 256         struct vc4_bo_cache *cache = &screen->bo_cache;
 257         uint32_t page_index = bo->size / 4096 - 1;
 258
 259         if (!bo->private) {
 260                 vc4_bo_free(bo);
 261                 return;
 262         }
 263
 264         if (cache->size_list_size <= page_index) {
 265                 struct list_head *new_list =
 266                         ralloc_array(screen, struct list_head, page_index + 1);
 267
 268                 /* Move old list contents over (since the array has moved, and
 269                  * therefore the pointers to the list heads have to change).
 270                  */
 271                 for (int i = 0; i < cache->size_list_size; i++) {
 272                         struct list_head *old_head = &cache->size_list[i];
 273                         if (list_empty(old_head))
 274                                 list_inithead(&new_list[i]);
 275                         else {
 276                                 new_list[i].next = old_head->next;
 277                                 new_list[i].prev = old_head->prev;
 278                                 new_list[i].next->prev = &new_list[i];
 279                                 new_list[i].prev->next = &new_list[i];
 280                         }
 281                 }
 282                 for (int i = cache->size_list_size; i < page_index + 1; i++)
 283                         list_inithead(&new_list[i]);
 284
 285                 cache->size_list = new_list;
 286                 cache->size_list_size = page_index + 1;
 287         }
 288
 289         bo->free_time = time;
 290         list_addtail(&bo->size_list, &cache->size_list[page_index]);
 291         list_addtail(&bo->time_list, &cache->time_list);
 292         cache->bo_count++;
 293         cache->bo_size += bo->size;
 294         if (dump_stats) {
 295                 fprintf(stderr, "Freed %s %dkb to cache:\n",
 296                         bo->name, bo->size / 1024);
 297                 vc4_bo_dump_stats(screen);
 298         }
 299         bo->name = NULL;
 300
 301         free_stale_bos(screen, time);
 302 }
 303
 304 static struct vc4_bo *
 305 vc4_bo_open_handle(struct vc4_screen *screen,
 306                    uint32_t winsys_stride,
 307                    uint32_t handle, uint32_t size)
 308 {
 309         struct vc4_bo *bo = CALLOC_STRUCT(vc4_bo);
 310
 311         assert(size);
 312
 313         pipe_reference_init(&bo->reference, 1);
 314         bo->screen = screen;
 315         bo->handle = handle;
 316         bo->size = size;
 317         bo->name = "winsys";
 318         bo->private = false;
 319
 320 #ifdef USE_VC4_SIMULATOR
 321         vc4_bo_map(bo);
 322         bo->simulator_winsys_map = bo->map;
 323         bo->simulator_winsys_stride = winsys_stride;
 324         bo->map = malloc(bo->size);
 325 #endif
 326
 327         return bo;
 328 }
 329
 330 struct vc4_bo *
 331 vc4_bo_open_name(struct vc4_screen *screen, uint32_t name,
 332                  uint32_t winsys_stride)
 333 {
 334         struct drm_gem_open o = {
 335                 .name = name
 336         };
 337         int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
 338         if (ret) {
 339                 fprintf(stderr, "Failed to open bo %d: %s\n",
 340                         name, strerror(errno));
 341                 return NULL;
 342         }
 343
 344         return vc4_bo_open_handle(screen, winsys_stride, o.handle, o.size);
 345 }
 346
 347 struct vc4_bo *
 348 vc4_bo_open_dmabuf(struct vc4_screen *screen, int fd, uint32_t winsys_stride)
 349 {
 350         uint32_t handle;
 351         int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
 352         int size;
 353         if (ret) {
 354                 fprintf(stderr, "Failed to get vc4 handle for dmabuf %d\n", fd);
 355                 return NULL;
 356         }
 357
 358         /* Determine the size of the bo we were handed. */
 359         size = lseek(fd, 0, SEEK_END);
 360         if (size == -1) {
 361                 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
 362                 return NULL;
 363         }
 364
 365         return vc4_bo_open_handle(screen, winsys_stride, handle, size);
 366 }
 367
 368 int
 369 vc4_bo_get_dmabuf(struct vc4_bo *bo)
 370 {
 371         int fd;
 372         int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
 373                                      O_CLOEXEC, &fd);
 374         if (ret != 0) {
 375                 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
 376                         bo->handle);
 377                 return -1;
 378         }
 379         bo->private = false;
 380
 381         return fd;
 382 }
 383
 384 struct vc4_bo *
 385 vc4_bo_alloc_mem(struct vc4_screen *screen, const void *data, uint32_t size,
 386                  const char *name)
 387 {
 388         void *map;
 389         struct vc4_bo *bo;
 390
 391         bo = vc4_bo_alloc(screen, size, name);
 392         map = vc4_bo_map(bo);
 393         memcpy(map, data, size);
 394         return bo;
 395 }
 396
 397 bool
 398 vc4_bo_flink(struct vc4_bo *bo, uint32_t *name)
 399 {
 400         struct drm_gem_flink flink = {
 401                 .handle = bo->handle,
 402         };
 403         int ret = drmIoctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
 404         if (ret) {
 405                 fprintf(stderr, "Failed to flink bo %d: %s\n",
 406                         bo->handle, strerror(errno));
 407                 free(bo);
 408                 return false;
 409         }
 410
 411         bo->private = false;
 412         *name = flink.name;
 413
 414         return true;
 415 }
 416
 417 bool
 418 vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns)
 419 {
 420         if (screen->finished_seqno >= seqno)
 421                 return true;
 422
 423         struct drm_vc4_wait_seqno wait;
 424         memset(&wait, 0, sizeof(wait));
 425         wait.seqno = seqno;
 426         wait.timeout_ns = timeout_ns;
 427
 428         int ret;
 429         if (!using_vc4_simulator)
 430                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait);
 431         else {
 432                 wait.seqno = screen->finished_seqno;
 433                 ret = 0;
 434         }
 435
 436         if (ret == 0) {
 437                 screen->finished_seqno = wait.seqno;
 438                 return true;
 439         }
 440
 441         if (errno != ETIME) {
 442                 fprintf(stderr, "wait failed: %d\n", ret);
 443                 abort();
 444         }
 445
 446         return false;
 447 }
 448
 449 bool
 450 vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns)
 451 {
 452         struct vc4_screen *screen = bo->screen;
 453
 454         struct drm_vc4_wait_bo wait;
 455         memset(&wait, 0, sizeof(wait));
 456         wait.handle = bo->handle;
 457         wait.timeout_ns = timeout_ns;
 458
 459         int ret;
 460         if (!using_vc4_simulator)
 461                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_WAIT_BO, &wait);
 462         else
 463                 ret = 0;
 464
 465         if (ret == 0)
 466                 return true;
 467
 468         if (errno != ETIME) {
 469                 fprintf(stderr, "wait failed: %d\n", ret);
 470                 abort();
 471         }
 472
 473         return false;
 474 }
 475
 476 void *
 477 vc4_bo_map_unsynchronized(struct vc4_bo *bo)
 478 {
 479         uint64_t offset;
 480         int ret;
 481
 482         if (bo->map)
 483                 return bo->map;
 484
 485         if (!using_vc4_simulator) {
 486                 struct drm_vc4_mmap_bo map;
 487                 memset(&map, 0, sizeof(map));
 488                 map.handle = bo->handle;
 489                 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_VC4_MMAP_BO, &map);
 490                 offset = map.offset;
 491         } else {
 492                 struct drm_mode_map_dumb map;
 493                 memset(&map, 0, sizeof(map));
 494                 map.handle = bo->handle;
 495                 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_MODE_MAP_DUMB, &map);
 496                 offset = map.offset;
 497         }
 498         if (ret != 0) {
 499                 fprintf(stderr, "map ioctl failure\n");
 500                 abort();
 501         }
 502
 503         bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
 504                        bo->screen->fd, offset);
 505         if (bo->map == MAP_FAILED) {
 506                 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
 507                         bo->handle, (long long)offset, bo->size);
 508                 abort();
 509         }
 510
 511         return bo->map;
 512 }
 513
 514 void *
 515 vc4_bo_map(struct vc4_bo *bo)
 516 {
 517         void *map = vc4_bo_map_unsynchronized(bo);
 518
 519         bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE);
 520         if (!ok) {
 521                 fprintf(stderr, "BO wait for map failed\n");
 522                 abort();
 523         }
 524
 525         return map;
 526 }
 527
 528 void
 529 vc4_bufmgr_destroy(struct pipe_screen *pscreen)
 530 {
 531         struct vc4_screen *screen = vc4_screen(pscreen);
 532         struct vc4_bo_cache *cache = &screen->bo_cache;
 533
 534         list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
 535                                  time_list) {
 536                 vc4_bo_remove_from_cache(cache, bo);
 537                 vc4_bo_free(bo);
 538         }
 539
 540         if (dump_stats) {
 541                 fprintf(stderr, "BO stats after screen destroy:\n");
 542                 vc4_bo_dump_stats(screen);
 543         }
 544 }