src/gallium/drivers/vc4/vc4_bufmgr.c

   1 /*
   2  * Copyright © 2014 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <errno.h>
  25 #include <err.h>
  26 #include <sys/mman.h>
  27 #include <fcntl.h>
  28 #include <xf86drm.h>
  29 #include <xf86drmMode.h>
  30
  31 #include "util/u_memory.h"
  32 #include "util/ralloc.h"
  33
  34 #include "vc4_context.h"
  35 #include "vc4_screen.h"
  36
  37 #define container_of(ptr, type, field) \
  38    (type*)((char*)ptr - offsetof(type, field))
  39
  40 static struct vc4_bo *
  41 vc4_bo_from_cache(struct vc4_screen *screen, uint32_t size, const char *name)
  42 {
  43         struct vc4_bo_cache *cache = &screen->bo_cache;
  44         uint32_t page_index = size / 4096 - 1;
  45
  46         if (cache->size_list_size <= page_index)
  47                 return NULL;
  48
  49         struct vc4_bo *bo = NULL;
  50         pipe_mutex_lock(cache->lock);
  51         if (!is_empty_list(&cache->size_list[page_index])) {
  52                 struct simple_node *node = first_elem(&cache->size_list[page_index]);
  53                 bo = container_of(node, struct vc4_bo, size_list);
  54
  55                 /* Check that the BO has gone idle.  If not, then we want to
  56                  * allocate something new instead, since we assume that the
  57                  * user will proceed to CPU map it and fill it with stuff.
  58                  */
  59                 if (!vc4_bo_wait(bo, 0)) {
  60                         pipe_mutex_unlock(cache->lock);
  61                         return NULL;
  62                 }
  63
  64                 pipe_reference_init(&bo->reference, 1);
  65                 remove_from_list(&bo->time_list);
  66                 remove_from_list(&bo->size_list);
  67
  68                 bo->name = name;
  69         }
  70         pipe_mutex_unlock(cache->lock);
  71         return bo;
  72 }
  73
  74 struct vc4_bo *
  75 vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name)
  76 {
  77         struct vc4_bo *bo;
  78         int ret;
  79
  80         size = align(size, 4096);
  81
  82         bo = vc4_bo_from_cache(screen, size, name);
  83         if (bo)
  84                 return bo;
  85
  86         bo = CALLOC_STRUCT(vc4_bo);
  87         if (!bo)
  88                 return NULL;
  89
  90         pipe_reference_init(&bo->reference, 1);
  91         bo->screen = screen;
  92         bo->size = size;
  93         bo->name = name;
  94         bo->private = true;
  95
  96         if (!using_vc4_simulator) {
  97                 struct drm_vc4_create_bo create;
  98                 memset(&create, 0, sizeof(create));
  99
 100                 create.size = size;
 101
 102                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_CREATE_BO, &create);
 103                 bo->handle = create.handle;
 104         } else {
 105                 struct drm_mode_create_dumb create;
 106                 memset(&create, 0, sizeof(create));
 107
 108                 create.width = 128;
 109                 create.bpp = 8;
 110                 create.height = (size + 127) / 128;
 111
 112                 ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
 113                 bo->handle = create.handle;
 114                 assert(create.size >= size);
 115         }
 116         if (ret != 0) {
 117                 fprintf(stderr, "create ioctl failure\n");
 118                 abort();
 119         }
 120
 121         return bo;
 122 }
 123
 124 void
 125 vc4_bo_last_unreference(struct vc4_bo *bo)
 126 {
 127         struct vc4_screen *screen = bo->screen;
 128
 129         struct timespec time;
 130         clock_gettime(CLOCK_MONOTONIC, &time);
 131         pipe_mutex_lock(screen->bo_cache.lock);
 132         vc4_bo_last_unreference_locked_timed(bo, time.tv_sec);
 133         pipe_mutex_unlock(screen->bo_cache.lock);
 134 }
 135
 136 static void
 137 vc4_bo_free(struct vc4_bo *bo)
 138 {
 139         struct vc4_screen *screen = bo->screen;
 140
 141         if (bo->map) {
 142 #ifdef USE_VC4_SIMULATOR
 143                 if (bo->simulator_winsys_map) {
 144                         free(bo->map);
 145                         bo->map = bo->simulator_winsys_map;
 146                 }
 147 #endif
 148                 munmap(bo->map, bo->size);
 149         }
 150
 151         struct drm_gem_close c;
 152         memset(&c, 0, sizeof(c));
 153         c.handle = bo->handle;
 154         int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
 155         if (ret != 0)
 156                 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
 157
 158         free(bo);
 159 }
 160
 161 static void
 162 free_stale_bos(struct vc4_screen *screen, time_t time)
 163 {
 164         while (!is_empty_list(&screen->bo_cache.time_list)) {
 165                 struct simple_node *node =
 166                         first_elem(&screen->bo_cache.time_list);
 167                 struct vc4_bo *bo = container_of(node, struct vc4_bo, time_list);
 168
 169                 /* If it's more than a second old, free it. */
 170                 if (time - bo->free_time > 2) {
 171                         remove_from_list(&bo->time_list);
 172                         remove_from_list(&bo->size_list);
 173                         vc4_bo_free(bo);
 174                 } else {
 175                         break;
 176                 }
 177         }
 178 }
 179
 180 void
 181 vc4_bo_last_unreference_locked_timed(struct vc4_bo *bo, time_t time)
 182 {
 183         struct vc4_screen *screen = bo->screen;
 184         struct vc4_bo_cache *cache = &screen->bo_cache;
 185         uint32_t page_index = bo->size / 4096 - 1;
 186
 187         if (!bo->private) {
 188                 vc4_bo_free(bo);
 189                 return;
 190         }
 191
 192         if (cache->size_list_size <= page_index) {
 193                 struct simple_node *new_list =
 194                         ralloc_array(screen, struct simple_node, page_index + 1);
 195
 196                 /* Move old list contents over (since the array has moved, and
 197                  * therefore the pointers to the list heads have to change.
 198                  */
 199                 for (int i = 0; i < cache->size_list_size; i++) {
 200                         struct simple_node *old_head = &cache->size_list[i];
 201                         if (is_empty_list(old_head))
 202                                 make_empty_list(&new_list[i]);
 203                         else {
 204                                 new_list[i].next = old_head->next;
 205                                 new_list[i].prev = old_head->prev;
 206                                 new_list[i].next->prev = &new_list[i];
 207                                 new_list[i].prev->next = &new_list[i];
 208                         }
 209                 }
 210                 for (int i = cache->size_list_size; i < page_index + 1; i++)
 211                         make_empty_list(&new_list[i]);
 212
 213                 cache->size_list = new_list;
 214                 cache->size_list_size = page_index + 1;
 215         }
 216
 217         bo->free_time = time;
 218         insert_at_tail(&cache->size_list[page_index], &bo->size_list);
 219         insert_at_tail(&cache->time_list, &bo->time_list);
 220
 221         free_stale_bos(screen, time);
 222 }
 223
 224 static struct vc4_bo *
 225 vc4_bo_open_handle(struct vc4_screen *screen,
 226                    uint32_t winsys_stride,
 227                    uint32_t handle, uint32_t size)
 228 {
 229         struct vc4_bo *bo = CALLOC_STRUCT(vc4_bo);
 230
 231         assert(size);
 232
 233         pipe_reference_init(&bo->reference, 1);
 234         bo->screen = screen;
 235         bo->handle = handle;
 236         bo->size = size;
 237         bo->name = "winsys";
 238         bo->private = false;
 239
 240 #ifdef USE_VC4_SIMULATOR
 241         vc4_bo_map(bo);
 242         bo->simulator_winsys_map = bo->map;
 243         bo->simulator_winsys_stride = winsys_stride;
 244         bo->map = malloc(bo->size);
 245 #endif
 246
 247         return bo;
 248 }
 249
 250 struct vc4_bo *
 251 vc4_bo_open_name(struct vc4_screen *screen, uint32_t name,
 252                  uint32_t winsys_stride)
 253 {
 254         struct drm_gem_open o = {
 255                 .name = name
 256         };
 257         int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
 258         if (ret) {
 259                 fprintf(stderr, "Failed to open bo %d: %s\n",
 260                         name, strerror(errno));
 261                 return NULL;
 262         }
 263
 264         return vc4_bo_open_handle(screen, winsys_stride, o.handle, o.size);
 265 }
 266
 267 struct vc4_bo *
 268 vc4_bo_open_dmabuf(struct vc4_screen *screen, int fd, uint32_t winsys_stride)
 269 {
 270         uint32_t handle;
 271         int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
 272         int size;
 273         if (ret) {
 274                 fprintf(stderr, "Failed to get vc4 handle for dmabuf %d\n", fd);
 275                 return NULL;
 276         }
 277
 278         /* Determine the size of the bo we were handed. */
 279         size = lseek(fd, 0, SEEK_END);
 280         if (size == -1) {
 281                 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
 282                 return NULL;
 283         }
 284
 285         return vc4_bo_open_handle(screen, winsys_stride, handle, size);
 286 }
 287
 288 int
 289 vc4_bo_get_dmabuf(struct vc4_bo *bo)
 290 {
 291         int fd;
 292         int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
 293                                      O_CLOEXEC, &fd);
 294         if (ret != 0) {
 295                 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
 296                         bo->handle);
 297                 return -1;
 298         }
 299         bo->private = false;
 300
 301         return fd;
 302 }
 303
 304 struct vc4_bo *
 305 vc4_bo_alloc_mem(struct vc4_screen *screen, const void *data, uint32_t size,
 306                  const char *name)
 307 {
 308         void *map;
 309         struct vc4_bo *bo;
 310
 311         bo = vc4_bo_alloc(screen, size, name);
 312         map = vc4_bo_map(bo);
 313         memcpy(map, data, size);
 314         return bo;
 315 }
 316
 317 bool
 318 vc4_bo_flink(struct vc4_bo *bo, uint32_t *name)
 319 {
 320         struct drm_gem_flink flink = {
 321                 .handle = bo->handle,
 322         };
 323         int ret = drmIoctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
 324         if (ret) {
 325                 fprintf(stderr, "Failed to flink bo %d: %s\n",
 326                         bo->handle, strerror(errno));
 327                 free(bo);
 328                 return false;
 329         }
 330
 331         bo->private = false;
 332         *name = flink.name;
 333
 334         return true;
 335 }
 336
 337 bool
 338 vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns)
 339 {
 340         if (screen->finished_seqno >= seqno)
 341                 return true;
 342
 343         struct drm_vc4_wait_seqno wait;
 344         memset(&wait, 0, sizeof(wait));
 345         wait.seqno = seqno;
 346         wait.timeout_ns = timeout_ns;
 347
 348         int ret;
 349         if (!using_vc4_simulator)
 350                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait);
 351         else {
 352                 wait.seqno = screen->finished_seqno;
 353                 ret = 0;
 354         }
 355
 356         if (ret == 0) {
 357                 screen->finished_seqno = wait.seqno;
 358                 return true;
 359         }
 360
 361         if (errno != ETIME) {
 362                 fprintf(stderr, "wait failed: %d\n", ret);
 363                 abort();
 364         }
 365
 366         return false;
 367 }
 368
 369 bool
 370 vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns)
 371 {
 372         struct vc4_screen *screen = bo->screen;
 373
 374         struct drm_vc4_wait_bo wait;
 375         memset(&wait, 0, sizeof(wait));
 376         wait.handle = bo->handle;
 377         wait.timeout_ns = timeout_ns;
 378
 379         int ret;
 380         if (!using_vc4_simulator)
 381                 ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_WAIT_BO, &wait);
 382         else
 383                 ret = 0;
 384
 385         if (ret == 0)
 386                 return true;
 387
 388         if (errno != ETIME) {
 389                 fprintf(stderr, "wait failed: %d\n", ret);
 390                 abort();
 391         }
 392
 393         return false;
 394 }
 395
 396 void *
 397 vc4_bo_map_unsynchronized(struct vc4_bo *bo)
 398 {
 399         uint64_t offset;
 400         int ret;
 401
 402         if (bo->map)
 403                 return bo->map;
 404
 405         if (!using_vc4_simulator) {
 406                 struct drm_vc4_mmap_bo map;
 407                 memset(&map, 0, sizeof(map));
 408                 map.handle = bo->handle;
 409                 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_VC4_MMAP_BO, &map);
 410                 offset = map.offset;
 411         } else {
 412                 struct drm_mode_map_dumb map;
 413                 memset(&map, 0, sizeof(map));
 414                 map.handle = bo->handle;
 415                 ret = drmIoctl(bo->screen->fd, DRM_IOCTL_MODE_MAP_DUMB, &map);
 416                 offset = map.offset;
 417         }
 418         if (ret != 0) {
 419                 fprintf(stderr, "map ioctl failure\n");
 420                 abort();
 421         }
 422
 423         bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
 424                        bo->screen->fd, offset);
 425         if (bo->map == MAP_FAILED) {
 426                 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
 427                         bo->handle, (long long)offset, bo->size);
 428                 abort();
 429         }
 430
 431         return bo->map;
 432 }
 433
 434 void *
 435 vc4_bo_map(struct vc4_bo *bo)
 436 {
 437         void *map = vc4_bo_map_unsynchronized(bo);
 438
 439         bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE);
 440         if (!ok) {
 441                 fprintf(stderr, "BO wait for map failed\n");
 442                 abort();
 443         }
 444
 445         return map;
 446 }
 447
 448 void
 449 vc4_bufmgr_destroy(struct pipe_screen *pscreen)
 450 {
 451         struct vc4_screen *screen = vc4_screen(pscreen);
 452         struct vc4_bo_cache *cache = &screen->bo_cache;
 453
 454         while (!is_empty_list(&cache->time_list)) {
 455                 struct simple_node *node = first_elem(&cache->time_list);
 456                 struct vc4_bo *bo = container_of(node, struct vc4_bo, time_list);
 457
 458                 remove_from_list(&bo->time_list);
 459                 remove_from_list(&bo->size_list);
 460                 vc4_bo_free(bo);
 461         }
 462 }