/*
- * Copyright © 2014 Broadcom
+ * Copyright © 2014-2015 Broadcom
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
#include <xf86drm.h>
#include <xf86drmMode.h>
+#include "util/u_hash_table.h"
#include "util/u_memory.h"
#include "util/ralloc.h"
#include "vc4_context.h"
#include "vc4_screen.h"
-#define container_of(ptr, type, field) \
- (type*)((char*)ptr - offsetof(type, field))
+#ifdef HAVE_VALGRIND
+#include <valgrind.h>
+#include <memcheck.h>
+#define VG(x) x
+#else
+#define VG(x)
+#endif
+
+static bool dump_stats = false;
+
+static void
+vc4_bo_cache_free_all(struct vc4_bo_cache *cache);
+
+static void
+vc4_bo_dump_stats(struct vc4_screen *screen)
+{
+ struct vc4_bo_cache *cache = &screen->bo_cache;
+
+ fprintf(stderr, " BOs allocated: %d\n", screen->bo_count);
+ fprintf(stderr, " BOs size: %dkb\n", screen->bo_size / 1024);
+ fprintf(stderr, " BOs cached: %d\n", cache->bo_count);
+ fprintf(stderr, " BOs cached size: %dkb\n", cache->bo_size / 1024);
+
+ if (!list_empty(&cache->time_list)) {
+ struct vc4_bo *first = LIST_ENTRY(struct vc4_bo,
+ cache->time_list.next,
+ time_list);
+ struct vc4_bo *last = LIST_ENTRY(struct vc4_bo,
+ cache->time_list.prev,
+ time_list);
+
+ fprintf(stderr, " oldest cache time: %ld\n",
+ (long)first->free_time);
+ fprintf(stderr, " newest cache time: %ld\n",
+ (long)last->free_time);
+
+ struct timespec time;
+ clock_gettime(CLOCK_MONOTONIC, &time);
+ fprintf(stderr, " now: %ld\n",
+ time.tv_sec);
+ }
+}
+
+static void
+vc4_bo_remove_from_cache(struct vc4_bo_cache *cache, struct vc4_bo *bo)
+{
+ list_del(&bo->time_list);
+ list_del(&bo->size_list);
+ cache->bo_count--;
+ cache->bo_size -= bo->size;
+}
static struct vc4_bo *
vc4_bo_from_cache(struct vc4_screen *screen, uint32_t size, const char *name)
return NULL;
struct vc4_bo *bo = NULL;
- pipe_mutex_lock(cache->lock);
- if (!is_empty_list(&cache->size_list[page_index])) {
- struct simple_node *node = last_elem(&cache->size_list[page_index]);
- bo = container_of(node, struct vc4_bo, size_list);
+ mtx_lock(&cache->lock);
+ if (!list_empty(&cache->size_list[page_index])) {
+ bo = LIST_ENTRY(struct vc4_bo, cache->size_list[page_index].next,
+ size_list);
+
+ /* Check that the BO has gone idle. If not, then we want to
+ * allocate something new instead, since we assume that the
+ * user will proceed to CPU map it and fill it with stuff.
+ */
+ if (!vc4_bo_wait(bo, 0, NULL)) {
+ mtx_unlock(&cache->lock);
+ return NULL;
+ }
+
pipe_reference_init(&bo->reference, 1);
- remove_from_list(&bo->time_list);
- remove_from_list(&bo->size_list);
+ vc4_bo_remove_from_cache(cache, bo);
bo->name = name;
}
- pipe_mutex_unlock(cache->lock);
+ mtx_unlock(&cache->lock);
return bo;
}
vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name)
{
struct vc4_bo *bo;
+ int ret;
+
size = align(size, 4096);
bo = vc4_bo_from_cache(screen, size, name);
- if (bo)
+ if (bo) {
+ if (dump_stats) {
+ fprintf(stderr, "Allocated %s %dkb from cache:\n",
+ name, size / 1024);
+ vc4_bo_dump_stats(screen);
+ }
return bo;
+ }
bo = CALLOC_STRUCT(vc4_bo);
if (!bo)
bo->name = name;
bo->private = true;
- struct drm_mode_create_dumb create;
- memset(&create, 0, sizeof(create));
+ retry:
+ ;
+
+ bool cleared_and_retried = false;
+ struct drm_vc4_create_bo create = {
+ .size = size
+ };
- create.width = 128;
- create.bpp = 8;
- create.height = (size + 127) / 128;
+ ret = vc4_ioctl(screen->fd, DRM_IOCTL_VC4_CREATE_BO, &create);
+ bo->handle = create.handle;
- int ret = drmIoctl(screen->fd, DRM_IOCTL_MODE_CREATE_DUMB, &create);
if (ret != 0) {
- fprintf(stderr, "create ioctl failure\n");
- abort();
+ if (!list_empty(&screen->bo_cache.time_list) &&
+ !cleared_and_retried) {
+ cleared_and_retried = true;
+ vc4_bo_cache_free_all(&screen->bo_cache);
+ goto retry;
+ }
+
+ free(bo);
+ return NULL;
}
- bo->handle = create.handle;
- assert(create.size >= size);
+ screen->bo_count++;
+ screen->bo_size += bo->size;
+ if (dump_stats) {
+ fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
+ vc4_bo_dump_stats(screen);
+ }
return bo;
}
struct timespec time;
clock_gettime(CLOCK_MONOTONIC, &time);
- pipe_mutex_lock(screen->bo_cache.lock);
+ mtx_lock(&screen->bo_cache.lock);
vc4_bo_last_unreference_locked_timed(bo, time.tv_sec);
- pipe_mutex_unlock(screen->bo_cache.lock);
+ mtx_unlock(&screen->bo_cache.lock);
}
static void
struct vc4_screen *screen = bo->screen;
if (bo->map) {
-#ifdef USE_VC4_SIMULATOR
- if (bo->simulator_winsys_map) {
+ if (using_vc4_simulator && bo->name &&
+ strcmp(bo->name, "winsys") == 0) {
free(bo->map);
- bo->map = bo->simulator_winsys_map;
+ } else {
+ munmap(bo->map, bo->size);
+ VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
}
-#endif
- munmap(bo->map, bo->size);
}
struct drm_gem_close c;
memset(&c, 0, sizeof(c));
c.handle = bo->handle;
- int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
+ int ret = vc4_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
if (ret != 0)
fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
+ screen->bo_count--;
+ screen->bo_size -= bo->size;
+
+ if (dump_stats) {
+ fprintf(stderr, "Freed %s%s%dkb:\n",
+ bo->name ? bo->name : "",
+ bo->name ? " " : "",
+ bo->size / 1024);
+ vc4_bo_dump_stats(screen);
+ }
+
free(bo);
}
static void
free_stale_bos(struct vc4_screen *screen, time_t time)
{
- while (!is_empty_list(&screen->bo_cache.time_list)) {
- struct simple_node *node =
- first_elem(&screen->bo_cache.time_list);
- struct vc4_bo *bo = container_of(node, struct vc4_bo, time_list);
+ struct vc4_bo_cache *cache = &screen->bo_cache;
+ bool freed_any = false;
+
+ list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
+ time_list) {
+ if (dump_stats && !freed_any) {
+ fprintf(stderr, "Freeing stale BOs:\n");
+ vc4_bo_dump_stats(screen);
+ freed_any = true;
+ }
/* If it's more than a second old, free it. */
if (time - bo->free_time > 2) {
- remove_from_list(&bo->time_list);
- remove_from_list(&bo->size_list);
+ vc4_bo_remove_from_cache(cache, bo);
vc4_bo_free(bo);
} else {
break;
}
}
+
+ if (dump_stats && freed_any) {
+ fprintf(stderr, "Freed stale BOs:\n");
+ vc4_bo_dump_stats(screen);
+ }
+}
+
+static void
+vc4_bo_cache_free_all(struct vc4_bo_cache *cache)
+{
+ mtx_lock(&cache->lock);
+ list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list,
+ time_list) {
+ vc4_bo_remove_from_cache(cache, bo);
+ vc4_bo_free(bo);
+ }
+ mtx_unlock(&cache->lock);
}
void
}
if (cache->size_list_size <= page_index) {
- struct simple_node *new_list =
- ralloc_array(screen, struct simple_node, page_index + 1);
+ struct list_head *new_list =
+ ralloc_array(screen, struct list_head, page_index + 1);
/* Move old list contents over (since the array has moved, and
- * therefore the pointers to the list heads have to change.
+ * therefore the pointers to the list heads have to change).
*/
- for (int i = 0; i < cache->size_list_size; i++) {
- struct simple_node *old_head = &cache->size_list[i];
- if (is_empty_list(old_head))
- make_empty_list(&new_list[i]);
- else {
- new_list[i].next = old_head->next;
- new_list[i].prev = old_head->prev;
- new_list[i].next->prev = &new_list[i];
- new_list[i].prev->next = &new_list[i];
- }
- }
+ for (int i = 0; i < cache->size_list_size; i++)
+ list_replace(&cache->size_list[i], &new_list[i]);
for (int i = cache->size_list_size; i < page_index + 1; i++)
- make_empty_list(&new_list[i]);
+ list_inithead(&new_list[i]);
cache->size_list = new_list;
cache->size_list_size = page_index + 1;
}
bo->free_time = time;
- insert_at_tail(&cache->size_list[page_index], &bo->size_list);
- insert_at_tail(&cache->time_list, &bo->time_list);
+ list_addtail(&bo->size_list, &cache->size_list[page_index]);
+ list_addtail(&bo->time_list, &cache->time_list);
+ cache->bo_count++;
+ cache->bo_size += bo->size;
+ if (dump_stats) {
+ fprintf(stderr, "Freed %s %dkb to cache:\n",
+ bo->name, bo->size / 1024);
+ vc4_bo_dump_stats(screen);
+ }
+ bo->name = NULL;
free_stale_bos(screen, time);
}
uint32_t winsys_stride,
uint32_t handle, uint32_t size)
{
- struct vc4_bo *bo = CALLOC_STRUCT(vc4_bo);
+ struct vc4_bo *bo;
assert(size);
+ mtx_lock(&screen->bo_handles_mutex);
+
+ bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
+ if (bo) {
+ pipe_reference(NULL, &bo->reference);
+ goto done;
+ }
+
+ bo = CALLOC_STRUCT(vc4_bo);
pipe_reference_init(&bo->reference, 1);
bo->screen = screen;
bo->handle = handle;
bo->private = false;
#ifdef USE_VC4_SIMULATOR
- vc4_bo_map(bo);
- bo->simulator_winsys_map = bo->map;
- bo->simulator_winsys_stride = winsys_stride;
+ vc4_simulator_open_from_handle(screen->fd, winsys_stride,
+ bo->handle, bo->size);
bo->map = malloc(bo->size);
#endif
+ util_hash_table_set(screen->bo_handles, (void *)(uintptr_t)handle, bo);
+
+done:
+ mtx_unlock(&screen->bo_handles_mutex);
return bo;
}
struct drm_gem_open o = {
.name = name
};
- int ret = drmIoctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
+ int ret = vc4_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
if (ret) {
fprintf(stderr, "Failed to open bo %d: %s\n",
name, strerror(errno));
return -1;
}
+ mtx_lock(&bo->screen->bo_handles_mutex);
+ bo->private = false;
+ util_hash_table_set(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
+ mtx_unlock(&bo->screen->bo_handles_mutex);
+
return fd;
}
struct vc4_bo *
-vc4_bo_alloc_mem(struct vc4_screen *screen, const void *data, uint32_t size,
- const char *name)
+vc4_bo_alloc_shader(struct vc4_screen *screen, const void *data, uint32_t size)
{
- void *map;
struct vc4_bo *bo;
+ int ret;
+
+ bo = CALLOC_STRUCT(vc4_bo);
+ if (!bo)
+ return NULL;
+
+ pipe_reference_init(&bo->reference, 1);
+ bo->screen = screen;
+ bo->size = align(size, 4096);
+ bo->name = "code";
+ bo->private = false; /* Make sure it doesn't go back to the cache. */
+
+ struct drm_vc4_create_shader_bo create = {
+ .size = size,
+ .data = (uintptr_t)data,
+ };
+
+ ret = vc4_ioctl(screen->fd, DRM_IOCTL_VC4_CREATE_SHADER_BO,
+ &create);
+ bo->handle = create.handle;
+
+ if (ret != 0) {
+ fprintf(stderr, "create shader ioctl failure\n");
+ abort();
+ }
+
+ screen->bo_count++;
+ screen->bo_size += bo->size;
+ if (dump_stats) {
+ fprintf(stderr, "Allocated shader %dkb:\n", bo->size / 1024);
+ vc4_bo_dump_stats(screen);
+ }
- bo = vc4_bo_alloc(screen, size, name);
- map = vc4_bo_map(bo);
- memcpy(map, data, size);
return bo;
}
struct drm_gem_flink flink = {
.handle = bo->handle,
};
- int ret = drmIoctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
+ int ret = vc4_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
if (ret) {
fprintf(stderr, "Failed to flink bo %d: %s\n",
bo->handle, strerror(errno));
return true;
}
+static int vc4_wait_seqno_ioctl(int fd, uint64_t seqno, uint64_t timeout_ns)
+{
+ struct drm_vc4_wait_seqno wait = {
+ .seqno = seqno,
+ .timeout_ns = timeout_ns,
+ };
+ int ret = vc4_ioctl(fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait);
+ if (ret == -1)
+ return -errno;
+ else
+ return 0;
+
+}
+
bool
-vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns)
+vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns,
+ const char *reason)
{
if (screen->finished_seqno >= seqno)
return true;
-#ifndef USE_VC4_SIMULATOR
- struct drm_vc4_wait_seqno wait;
- memset(&wait, 0, sizeof(wait));
- wait.seqno = seqno;
- wait.timeout_ns = timeout_ns;
+ if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
+ if (vc4_wait_seqno_ioctl(screen->fd, seqno, 0) == -ETIME) {
+ fprintf(stderr, "Blocking on seqno %lld for %s\n",
+ (long long)seqno, reason);
+ }
+ }
+
+ int ret = vc4_wait_seqno_ioctl(screen->fd, seqno, timeout_ns);
+ if (ret) {
+ if (ret != -ETIME) {
+ fprintf(stderr, "wait failed: %d\n", ret);
+ abort();
+ }
- int ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait);
- if (ret == -ETIME) {
return false;
- } else if (ret != 0) {
- fprintf(stderr, "wait failed\n");
- abort();
- } else {
- screen->finished_seqno = wait.seqno;
- return true;
}
-#else
+
+ screen->finished_seqno = seqno;
return true;
-#endif
+}
+
+static int vc4_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
+{
+ struct drm_vc4_wait_bo wait = {
+ .handle = handle,
+ .timeout_ns = timeout_ns,
+ };
+ int ret = vc4_ioctl(fd, DRM_IOCTL_VC4_WAIT_BO, &wait);
+ if (ret == -1)
+ return -errno;
+ else
+ return 0;
+
}
bool
-vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns)
+vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns, const char *reason)
{
-#ifndef USE_VC4_SIMULATOR
struct vc4_screen *screen = bo->screen;
- struct drm_vc4_wait_bo wait;
- memset(&wait, 0, sizeof(wait));
- wait.handle = bo->handle;
- wait.timeout_ns = timeout_ns;
+ if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) {
+ if (vc4_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
+ fprintf(stderr, "Blocking on %s BO for %s\n",
+ bo->name, reason);
+ }
+ }
+
+ int ret = vc4_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
+ if (ret) {
+ if (ret != -ETIME) {
+ fprintf(stderr, "wait failed: %d\n", ret);
+ abort();
+ }
- int ret = drmIoctl(screen->fd, DRM_IOCTL_VC4_WAIT_BO, &wait);
- if (ret == -ETIME) {
return false;
- } else if (ret != 0) {
- fprintf(stderr, "wait failed\n");
- abort();
- } else {
- return true;
}
-#else
+
return true;
-#endif
}
void *
vc4_bo_map_unsynchronized(struct vc4_bo *bo)
{
+ uint64_t offset;
int ret;
if (bo->map)
return bo->map;
- struct drm_mode_map_dumb map;
+ struct drm_vc4_mmap_bo map;
memset(&map, 0, sizeof(map));
map.handle = bo->handle;
- ret = drmIoctl(bo->screen->fd, DRM_IOCTL_MODE_MAP_DUMB, &map);
+ ret = vc4_ioctl(bo->screen->fd, DRM_IOCTL_VC4_MMAP_BO, &map);
+ offset = map.offset;
if (ret != 0) {
fprintf(stderr, "map ioctl failure\n");
abort();
}
bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
- bo->screen->fd, map.offset);
+ bo->screen->fd, offset);
if (bo->map == MAP_FAILED) {
fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
- bo->handle, (long long)map.offset, bo->size);
+ bo->handle, (long long)offset, bo->size);
abort();
}
+ VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false));
return bo->map;
}
{
void *map = vc4_bo_map_unsynchronized(bo);
- bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE);
+ bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map");
if (!ok) {
fprintf(stderr, "BO wait for map failed\n");
abort();
struct vc4_screen *screen = vc4_screen(pscreen);
struct vc4_bo_cache *cache = &screen->bo_cache;
- while (!is_empty_list(&cache->time_list)) {
- struct simple_node *node = first_elem(&cache->time_list);
- struct vc4_bo *bo = container_of(node, struct vc4_bo, time_list);
+ vc4_bo_cache_free_all(cache);
- remove_from_list(&bo->time_list);
- remove_from_list(&bo->size_list);
- vc4_bo_free(bo);
+ if (dump_stats) {
+ fprintf(stderr, "BO stats after screen destroy:\n");
+ vc4_bo_dump_stats(screen);
}
}
projects / mesa.git / blobdiff