X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fvc4%2Fvc4_bufmgr.c;h=b300fa5bd025e43d91e2e1ff48637a2332d50d12;hb=44993bd26ff3d8da503f3bb2385c93960bf6fbb7;hp=581ba89fd07f12494ac606157b941c910d4ba58c;hpb=cdc208bdaf90017c2e1aaa54d2318b956e801ca0;p=mesa.git diff --git a/src/gallium/drivers/vc4/vc4_bufmgr.c b/src/gallium/drivers/vc4/vc4_bufmgr.c index 581ba89fd07..b300fa5bd02 100644 --- a/src/gallium/drivers/vc4/vc4_bufmgr.c +++ b/src/gallium/drivers/vc4/vc4_bufmgr.c @@ -1,5 +1,5 @@ /* - * Copyright © 2014 Broadcom + * Copyright © 2014-2015 Broadcom * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -23,20 +23,151 @@ #include #include -#include #include +#include #include #include +#include "util/u_hash_table.h" #include "util/u_memory.h" +#include "util/ralloc.h" #include "vc4_context.h" #include "vc4_screen.h" +#ifdef HAVE_VALGRIND +#include +#include +#define VG(x) x +#else +#define VG(x) +#endif + +static bool dump_stats = false; + +static void +vc4_bo_cache_free_all(struct vc4_bo_cache *cache); + +void +vc4_bo_label(struct vc4_screen *screen, struct vc4_bo *bo, const char *fmt, ...) +{ + /* Perform BO labeling by default on debug builds (so that you get + * whole-system allocation information), or if VC4_DEBUG=surf is set + * (for debugging a single app's allocation). + */ +#ifndef DEBUG + if (!(vc4_debug & VC4_DEBUG_SURFACE)) + return; +#endif + va_list va; + va_start(va, fmt); + char *name = ralloc_vasprintf(NULL, fmt, va); + va_end(va); + + struct drm_vc4_label_bo label = { + .handle = bo->handle, + .len = strlen(name), + .name = (uintptr_t)name, + }; + drmIoctl(screen->fd, DRM_IOCTL_VC4_LABEL_BO, &label); + + ralloc_free(name); +} + +static void +vc4_bo_dump_stats(struct vc4_screen *screen) +{ + struct vc4_bo_cache *cache = &screen->bo_cache; + + fprintf(stderr, " BOs allocated: %d\n", screen->bo_count); + fprintf(stderr, " BOs size: %dkb\n", screen->bo_size / 1024); + fprintf(stderr, " BOs cached: %d\n", cache->bo_count); + fprintf(stderr, " BOs cached size: %dkb\n", cache->bo_size / 1024); + + if (!list_empty(&cache->time_list)) { + struct vc4_bo *first = LIST_ENTRY(struct vc4_bo, + cache->time_list.next, + time_list); + struct vc4_bo *last = LIST_ENTRY(struct vc4_bo, + cache->time_list.prev, + time_list); + + fprintf(stderr, " oldest cache time: %ld\n", + (long)first->free_time); + fprintf(stderr, " newest cache time: %ld\n", + (long)last->free_time); + + struct timespec time; + clock_gettime(CLOCK_MONOTONIC, &time); + fprintf(stderr, " now: %ld\n", + time.tv_sec); + } +} + +static void +vc4_bo_remove_from_cache(struct vc4_bo_cache *cache, struct vc4_bo *bo) +{ + list_del(&bo->time_list); + list_del(&bo->size_list); + cache->bo_count--; + cache->bo_size -= bo->size; +} + +static struct vc4_bo * +vc4_bo_from_cache(struct vc4_screen *screen, uint32_t size, const char *name) +{ + struct vc4_bo_cache *cache = &screen->bo_cache; + uint32_t page_index = size / 4096 - 1; + + if (cache->size_list_size <= page_index) + return NULL; + + struct vc4_bo *bo = NULL; + mtx_lock(&cache->lock); + if (!list_empty(&cache->size_list[page_index])) { + bo = LIST_ENTRY(struct vc4_bo, cache->size_list[page_index].next, + size_list); + + /* Check that the BO has gone idle. If not, then we want to + * allocate something new instead, since we assume that the + * user will proceed to CPU map it and fill it with stuff. + */ + if (!vc4_bo_wait(bo, 0, NULL)) { + mtx_unlock(&cache->lock); + return NULL; + } + + pipe_reference_init(&bo->reference, 1); + vc4_bo_remove_from_cache(cache, bo); + + vc4_bo_label(screen, bo, "%s", name); + bo->name = name; + } + mtx_unlock(&cache->lock); + return bo; +} + struct vc4_bo * vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name) { - struct vc4_bo *bo = CALLOC_STRUCT(vc4_bo); + bool cleared_and_retried = false; + struct drm_vc4_create_bo create; + struct vc4_bo *bo; + int ret; + + size = align(size, 4096); + + bo = vc4_bo_from_cache(screen, size, name); + if (bo) { + if (dump_stats) { + fprintf(stderr, "Allocated %s %dkb from cache:\n", + name, size / 1024); + vc4_bo_dump_stats(screen); + } + return bo; + } + + bo = CALLOC_STRUCT(vc4_bo); if (!bo) return NULL; @@ -44,104 +175,315 @@ vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name) bo->screen = screen; bo->size = size; bo->name = name; + bo->private = true; -#ifndef USE_VC4_SIMULATOR - struct drm_mode_create_dumb create; + retry: memset(&create, 0, sizeof(create)); + create.size = size; - create.width = 128; - create.bpp = 8; - create.height = (size + 127) / 128; + ret = vc4_ioctl(screen->fd, DRM_IOCTL_VC4_CREATE_BO, &create); + bo->handle = create.handle; - int ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create); - if (ret != 0) - errx(1, "create ioctl"); + if (ret != 0) { + if (!list_empty(&screen->bo_cache.time_list) && + !cleared_and_retried) { + cleared_and_retried = true; + vc4_bo_cache_free_all(&screen->bo_cache); + goto retry; + } - bo->handle = create.handle; - assert(create.size >= size); -#else /* USE_VC4_SIMULATOR */ - static int next_handle = 0; - bo->handle = next_handle++; + free(bo); + return NULL; + } - bo->map = malloc(size); -#endif /* USE_VC4_SIMULATOR */ + screen->bo_count++; + screen->bo_size += bo->size; + if (dump_stats) { + fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024); + vc4_bo_dump_stats(screen); + } + + vc4_bo_label(screen, bo, "%s", name); return bo; } void +vc4_bo_last_unreference(struct vc4_bo *bo) +{ + struct vc4_screen *screen = bo->screen; + + struct timespec time; + clock_gettime(CLOCK_MONOTONIC, &time); + mtx_lock(&screen->bo_cache.lock); + vc4_bo_last_unreference_locked_timed(bo, time.tv_sec); + mtx_unlock(&screen->bo_cache.lock); +} + +static void vc4_bo_free(struct vc4_bo *bo) { -#ifndef USE_VC4_SIMULATOR struct vc4_screen *screen = bo->screen; - if (bo->map) - munmap(bo->map, bo->size); + if (bo->map) { + if (using_vc4_simulator && bo->name && + strcmp(bo->name, "winsys") == 0) { + free(bo->map); + } else { + munmap(bo->map, bo->size); + VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0)); + } + } struct drm_gem_close c; + memset(&c, 0, sizeof(c)); c.handle = bo->handle; - int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c); + int ret = vc4_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c); if (ret != 0) fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno)); -#else - free(bo->map); -#endif + + screen->bo_count--; + screen->bo_size -= bo->size; + + if (dump_stats) { + fprintf(stderr, "Freed %s%s%dkb:\n", + bo->name ? bo->name : "", + bo->name ? " " : "", + bo->size / 1024); + vc4_bo_dump_stats(screen); + } free(bo); } -struct vc4_bo * -vc4_bo_open_name(struct vc4_screen *screen, uint32_t name, - uint32_t winsys_stride) +static void +free_stale_bos(struct vc4_screen *screen, time_t time) { - struct vc4_bo *bo = CALLOC_STRUCT(vc4_bo); + struct vc4_bo_cache *cache = &screen->bo_cache; + bool freed_any = false; - struct drm_gem_open o; - o.name = name; - int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o); - if (ret) { - fprintf(stderr, "Failed to open bo %d: %s\n", - name, strerror(errno)); - free(bo); - return NULL; + list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list, + time_list) { + if (dump_stats && !freed_any) { + fprintf(stderr, "Freeing stale BOs:\n"); + vc4_bo_dump_stats(screen); + freed_any = true; + } + + /* If it's more than a second old, free it. */ + if (time - bo->free_time > 2) { + vc4_bo_remove_from_cache(cache, bo); + vc4_bo_free(bo); + } else { + break; + } + } + + if (dump_stats && freed_any) { + fprintf(stderr, "Freed stale BOs:\n"); + vc4_bo_dump_stats(screen); + } +} + +static void +vc4_bo_cache_free_all(struct vc4_bo_cache *cache) +{ + mtx_lock(&cache->lock); + list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list, + time_list) { + vc4_bo_remove_from_cache(cache, bo); + vc4_bo_free(bo); } + mtx_unlock(&cache->lock); +} +void +vc4_bo_last_unreference_locked_timed(struct vc4_bo *bo, time_t time) +{ + struct vc4_screen *screen = bo->screen; + struct vc4_bo_cache *cache = &screen->bo_cache; + uint32_t page_index = bo->size / 4096 - 1; + + if (!bo->private) { + vc4_bo_free(bo); + return; + } + + if (cache->size_list_size <= page_index) { + struct list_head *new_list = + ralloc_array(screen, struct list_head, page_index + 1); + + /* Move old list contents over (since the array has moved, and + * therefore the pointers to the list heads have to change). + */ + for (int i = 0; i < cache->size_list_size; i++) + list_replace(&cache->size_list[i], &new_list[i]); + for (int i = cache->size_list_size; i < page_index + 1; i++) + list_inithead(&new_list[i]); + + cache->size_list = new_list; + cache->size_list_size = page_index + 1; + } + + bo->free_time = time; + list_addtail(&bo->size_list, &cache->size_list[page_index]); + list_addtail(&bo->time_list, &cache->time_list); + cache->bo_count++; + cache->bo_size += bo->size; + if (dump_stats) { + fprintf(stderr, "Freed %s %dkb to cache:\n", + bo->name, bo->size / 1024); + vc4_bo_dump_stats(screen); + } + bo->name = NULL; + vc4_bo_label(screen, bo, "mesa cache"); + + free_stale_bos(screen, time); +} + +static struct vc4_bo * +vc4_bo_open_handle(struct vc4_screen *screen, + uint32_t winsys_stride, + uint32_t handle, uint32_t size) +{ + struct vc4_bo *bo; + + assert(size); + + mtx_lock(&screen->bo_handles_mutex); + + bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle); + if (bo) { + pipe_reference(NULL, &bo->reference); + goto done; + } + + bo = CALLOC_STRUCT(vc4_bo); pipe_reference_init(&bo->reference, 1); bo->screen = screen; - bo->handle = o.handle; - bo->size = o.size; + bo->handle = handle; + bo->size = size; + bo->name = "winsys"; + bo->private = false; #ifdef USE_VC4_SIMULATOR - vc4_bo_map(bo); - bo->simulator_winsys_map = bo->map; - bo->simulator_winsys_stride = winsys_stride; + vc4_simulator_open_from_handle(screen->fd, winsys_stride, + bo->handle, bo->size); bo->map = malloc(bo->size); #endif + util_hash_table_set(screen->bo_handles, (void *)(uintptr_t)handle, bo); + +done: + mtx_unlock(&screen->bo_handles_mutex); return bo; } struct vc4_bo * -vc4_bo_alloc_mem(struct vc4_screen *screen, const void *data, uint32_t size, - const char *name) +vc4_bo_open_name(struct vc4_screen *screen, uint32_t name, + uint32_t winsys_stride) +{ + struct drm_gem_open o = { + .name = name + }; + int ret = vc4_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o); + if (ret) { + fprintf(stderr, "Failed to open bo %d: %s\n", + name, strerror(errno)); + return NULL; + } + + return vc4_bo_open_handle(screen, winsys_stride, o.handle, o.size); +} + +struct vc4_bo * +vc4_bo_open_dmabuf(struct vc4_screen *screen, int fd, uint32_t winsys_stride) +{ + uint32_t handle; + int ret = drmPrimeFDToHandle(screen->fd, fd, &handle); + int size; + if (ret) { + fprintf(stderr, "Failed to get vc4 handle for dmabuf %d\n", fd); + return NULL; + } + + /* Determine the size of the bo we were handed. */ + size = lseek(fd, 0, SEEK_END); + if (size == -1) { + fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd); + return NULL; + } + + return vc4_bo_open_handle(screen, winsys_stride, handle, size); +} + +int +vc4_bo_get_dmabuf(struct vc4_bo *bo) +{ + int fd; + int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle, + O_CLOEXEC, &fd); + if (ret != 0) { + fprintf(stderr, "Failed to export gem bo %d to dmabuf\n", + bo->handle); + return -1; + } + + mtx_lock(&bo->screen->bo_handles_mutex); + bo->private = false; + util_hash_table_set(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo); + mtx_unlock(&bo->screen->bo_handles_mutex); + + return fd; +} + +struct vc4_bo * +vc4_bo_alloc_shader(struct vc4_screen *screen, const void *data, uint32_t size) { - void *map; struct vc4_bo *bo; + int ret; + + bo = CALLOC_STRUCT(vc4_bo); + if (!bo) + return NULL; + + pipe_reference_init(&bo->reference, 1); + bo->screen = screen; + bo->size = align(size, 4096); + bo->name = "code"; + bo->private = false; /* Make sure it doesn't go back to the cache. */ + + struct drm_vc4_create_shader_bo create = { + .size = size, + .data = (uintptr_t)data, + }; + + ret = vc4_ioctl(screen->fd, DRM_IOCTL_VC4_CREATE_SHADER_BO, + &create); + bo->handle = create.handle; + + if (ret != 0) { + fprintf(stderr, "create shader ioctl failure\n"); + abort(); + } + + screen->bo_count++; + screen->bo_size += bo->size; + if (dump_stats) { + fprintf(stderr, "Allocated shader %dkb:\n", bo->size / 1024); + vc4_bo_dump_stats(screen); + } - bo = vc4_bo_alloc(screen, size, name); - map = vc4_bo_map(bo); - memcpy(map, data, size); return bo; } bool vc4_bo_flink(struct vc4_bo *bo, uint32_t *name) { -#ifndef USE_VC4_SIMULATOR struct drm_gem_flink flink = { .handle = bo->handle, }; - int ret = drmIoctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink); + int ret = vc4_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink); if (ret) { fprintf(stderr, "Failed to flink bo %d: %s\n", bo->handle, strerror(errno)); @@ -149,33 +491,148 @@ vc4_bo_flink(struct vc4_bo *bo, uint32_t *name) return false; } + bo->private = false; *name = flink.name; -#endif /* USE_VC4_SIMULATOR */ + + return true; +} + +static int vc4_wait_seqno_ioctl(int fd, uint64_t seqno, uint64_t timeout_ns) +{ + struct drm_vc4_wait_seqno wait = { + .seqno = seqno, + .timeout_ns = timeout_ns, + }; + int ret = vc4_ioctl(fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait); + if (ret == -1) + return -errno; + else + return 0; + +} + +bool +vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns, + const char *reason) +{ + if (screen->finished_seqno >= seqno) + return true; + + if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) { + if (vc4_wait_seqno_ioctl(screen->fd, seqno, 0) == -ETIME) { + fprintf(stderr, "Blocking on seqno %lld for %s\n", + (long long)seqno, reason); + } + } + + int ret = vc4_wait_seqno_ioctl(screen->fd, seqno, timeout_ns); + if (ret) { + if (ret != -ETIME) { + fprintf(stderr, "wait failed: %d\n", ret); + abort(); + } + + return false; + } + + screen->finished_seqno = seqno; + return true; +} + +static int vc4_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns) +{ + struct drm_vc4_wait_bo wait = { + .handle = handle, + .timeout_ns = timeout_ns, + }; + int ret = vc4_ioctl(fd, DRM_IOCTL_VC4_WAIT_BO, &wait); + if (ret == -1) + return -errno; + else + return 0; + +} + +bool +vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns, const char *reason) +{ + struct vc4_screen *screen = bo->screen; + + if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) { + if (vc4_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) { + fprintf(stderr, "Blocking on %s BO for %s\n", + bo->name, reason); + } + } + + int ret = vc4_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns); + if (ret) { + if (ret != -ETIME) { + fprintf(stderr, "wait failed: %d\n", ret); + abort(); + } + + return false; + } return true; } void * -vc4_bo_map(struct vc4_bo *bo) +vc4_bo_map_unsynchronized(struct vc4_bo *bo) { + uint64_t offset; int ret; if (bo->map) return bo->map; - struct drm_mode_map_dumb map; + struct drm_vc4_mmap_bo map; memset(&map, 0, sizeof(map)); map.handle = bo->handle; - ret = drmIoctl(bo->screen->fd, DRM_IOCTL_MODE_MAP_DUMB, &map); - if (ret != 0) - errx(1, "map ioctl"); + ret = vc4_ioctl(bo->screen->fd, DRM_IOCTL_VC4_MMAP_BO, &map); + offset = map.offset; + if (ret != 0) { + fprintf(stderr, "map ioctl failure\n"); + abort(); + } bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED, - bo->screen->fd, map.offset); + bo->screen->fd, offset); if (bo->map == MAP_FAILED) { - errx(1, "mmap of bo %d (offset 0x%016llx, size %d) failed\n", - bo->handle, (long long)map.offset, bo->size); + fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n", + bo->handle, (long long)offset, bo->size); + abort(); } + VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false)); return bo->map; } + +void * +vc4_bo_map(struct vc4_bo *bo) +{ + void *map = vc4_bo_map_unsynchronized(bo); + + bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map"); + if (!ok) { + fprintf(stderr, "BO wait for map failed\n"); + abort(); + } + + return map; +} + +void +vc4_bufmgr_destroy(struct pipe_screen *pscreen) +{ + struct vc4_screen *screen = vc4_screen(pscreen); + struct vc4_bo_cache *cache = &screen->bo_cache; + + vc4_bo_cache_free_all(cache); + + if (dump_stats) { + fprintf(stderr, "BO stats after screen destroy:\n"); + vc4_bo_dump_stats(screen); + } +}