2 * Copyright © 2014-2017 Broadcom
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
29 #include <xf86drmMode.h>
31 #include "util/u_hash_table.h"
32 #include "util/u_memory.h"
33 #include "util/ralloc.h"
35 #include "v3d_context.h"
36 #include "v3d_screen.h"
46 static bool dump_stats
= false;
49 v3d_bo_cache_free_all(struct v3d_bo_cache
*cache
);
52 v3d_bo_dump_stats(struct v3d_screen
*screen
)
54 struct v3d_bo_cache
*cache
= &screen
->bo_cache
;
56 uint32_t cache_count
= 0;
57 uint32_t cache_size
= 0;
58 list_for_each_entry(struct v3d_bo
, bo
, &cache
->time_list
, time_list
) {
60 cache_size
+= bo
->size
;
63 fprintf(stderr
, " BOs allocated: %d\n", screen
->bo_count
);
64 fprintf(stderr
, " BOs size: %dkb\n", screen
->bo_size
/ 1024);
65 fprintf(stderr
, " BOs cached: %d\n", cache_count
);
66 fprintf(stderr
, " BOs cached size: %dkb\n", cache_size
/ 1024);
68 if (!list_empty(&cache
->time_list
)) {
69 struct v3d_bo
*first
= list_first_entry(&cache
->time_list
,
72 struct v3d_bo
*last
= list_last_entry(&cache
->time_list
,
76 fprintf(stderr
, " oldest cache time: %ld\n",
77 (long)first
->free_time
);
78 fprintf(stderr
, " newest cache time: %ld\n",
79 (long)last
->free_time
);
82 clock_gettime(CLOCK_MONOTONIC
, &time
);
83 fprintf(stderr
, " now: %ld\n",
89 v3d_bo_remove_from_cache(struct v3d_bo_cache
*cache
, struct v3d_bo
*bo
)
91 list_del(&bo
->time_list
);
92 list_del(&bo
->size_list
);
95 static struct v3d_bo
*
96 v3d_bo_from_cache(struct v3d_screen
*screen
, uint32_t size
, const char *name
)
98 struct v3d_bo_cache
*cache
= &screen
->bo_cache
;
99 uint32_t page_index
= size
/ 4096 - 1;
101 if (cache
->size_list_size
<= page_index
)
104 struct v3d_bo
*bo
= NULL
;
105 mtx_lock(&cache
->lock
);
106 if (!list_empty(&cache
->size_list
[page_index
])) {
107 bo
= list_first_entry(&cache
->size_list
[page_index
],
108 struct v3d_bo
, size_list
);
110 /* Check that the BO has gone idle. If not, then we want to
111 * allocate something new instead, since we assume that the
112 * user will proceed to CPU map it and fill it with stuff.
114 if (!v3d_bo_wait(bo
, 0, NULL
)) {
115 mtx_unlock(&cache
->lock
);
119 pipe_reference_init(&bo
->reference
, 1);
120 v3d_bo_remove_from_cache(cache
, bo
);
124 mtx_unlock(&cache
->lock
);
129 v3d_bo_alloc(struct v3d_screen
*screen
, uint32_t size
, const char *name
)
134 /* The CLIF dumping requires that there is no whitespace in the name.
136 assert(!strchr(name
, ' '));
138 size
= align(size
, 4096);
140 bo
= v3d_bo_from_cache(screen
, size
, name
);
143 fprintf(stderr
, "Allocated %s %dkb from cache:\n",
145 v3d_bo_dump_stats(screen
);
150 bo
= CALLOC_STRUCT(v3d_bo
);
154 pipe_reference_init(&bo
->reference
, 1);
163 bool cleared_and_retried
= false;
164 struct drm_v3d_create_bo create
= {
168 ret
= v3d_ioctl(screen
->fd
, DRM_IOCTL_V3D_CREATE_BO
, &create
);
169 bo
->handle
= create
.handle
;
170 bo
->offset
= create
.offset
;
173 if (!list_empty(&screen
->bo_cache
.time_list
) &&
174 !cleared_and_retried
) {
175 cleared_and_retried
= true;
176 v3d_bo_cache_free_all(&screen
->bo_cache
);
185 screen
->bo_size
+= bo
->size
;
187 fprintf(stderr
, "Allocated %s %dkb:\n", name
, size
/ 1024);
188 v3d_bo_dump_stats(screen
);
195 v3d_bo_last_unreference(struct v3d_bo
*bo
)
197 struct v3d_screen
*screen
= bo
->screen
;
199 struct timespec time
;
200 clock_gettime(CLOCK_MONOTONIC
, &time
);
201 mtx_lock(&screen
->bo_cache
.lock
);
202 v3d_bo_last_unreference_locked_timed(bo
, time
.tv_sec
);
203 mtx_unlock(&screen
->bo_cache
.lock
);
207 v3d_bo_free(struct v3d_bo
*bo
)
209 struct v3d_screen
*screen
= bo
->screen
;
212 if (using_v3d_simulator
&& bo
->name
&&
213 strcmp(bo
->name
, "winsys") == 0) {
216 munmap(bo
->map
, bo
->size
);
217 VG(VALGRIND_FREELIKE_BLOCK(bo
->map
, 0));
221 struct drm_gem_close c
;
222 memset(&c
, 0, sizeof(c
));
223 c
.handle
= bo
->handle
;
224 int ret
= v3d_ioctl(screen
->fd
, DRM_IOCTL_GEM_CLOSE
, &c
);
226 fprintf(stderr
, "close object %d: %s\n", bo
->handle
, strerror(errno
));
229 screen
->bo_size
-= bo
->size
;
232 fprintf(stderr
, "Freed %s%s%dkb:\n",
233 bo
->name
? bo
->name
: "",
236 v3d_bo_dump_stats(screen
);
243 free_stale_bos(struct v3d_screen
*screen
, time_t time
)
245 struct v3d_bo_cache
*cache
= &screen
->bo_cache
;
246 bool freed_any
= false;
248 list_for_each_entry_safe(struct v3d_bo
, bo
, &cache
->time_list
,
250 /* If it's more than a second old, free it. */
251 if (time
- bo
->free_time
> 2) {
252 if (dump_stats
&& !freed_any
) {
253 fprintf(stderr
, "Freeing stale BOs:\n");
254 v3d_bo_dump_stats(screen
);
257 v3d_bo_remove_from_cache(cache
, bo
);
264 if (dump_stats
&& freed_any
) {
265 fprintf(stderr
, "Freed stale BOs:\n");
266 v3d_bo_dump_stats(screen
);
271 v3d_bo_cache_free_all(struct v3d_bo_cache
*cache
)
273 mtx_lock(&cache
->lock
);
274 list_for_each_entry_safe(struct v3d_bo
, bo
, &cache
->time_list
,
276 v3d_bo_remove_from_cache(cache
, bo
);
279 mtx_unlock(&cache
->lock
);
283 v3d_bo_last_unreference_locked_timed(struct v3d_bo
*bo
, time_t time
)
285 struct v3d_screen
*screen
= bo
->screen
;
286 struct v3d_bo_cache
*cache
= &screen
->bo_cache
;
287 uint32_t page_index
= bo
->size
/ 4096 - 1;
294 if (cache
->size_list_size
<= page_index
) {
295 struct list_head
*new_list
=
296 ralloc_array(screen
, struct list_head
, page_index
+ 1);
298 /* Move old list contents over (since the array has moved, and
299 * therefore the pointers to the list heads have to change).
301 for (int i
= 0; i
< cache
->size_list_size
; i
++) {
302 struct list_head
*old_head
= &cache
->size_list
[i
];
303 if (list_empty(old_head
))
304 list_inithead(&new_list
[i
]);
306 new_list
[i
].next
= old_head
->next
;
307 new_list
[i
].prev
= old_head
->prev
;
308 new_list
[i
].next
->prev
= &new_list
[i
];
309 new_list
[i
].prev
->next
= &new_list
[i
];
312 for (int i
= cache
->size_list_size
; i
< page_index
+ 1; i
++)
313 list_inithead(&new_list
[i
]);
315 cache
->size_list
= new_list
;
316 cache
->size_list_size
= page_index
+ 1;
319 bo
->free_time
= time
;
320 list_addtail(&bo
->size_list
, &cache
->size_list
[page_index
]);
321 list_addtail(&bo
->time_list
, &cache
->time_list
);
323 fprintf(stderr
, "Freed %s %dkb to cache:\n",
324 bo
->name
, bo
->size
/ 1024);
325 v3d_bo_dump_stats(screen
);
329 free_stale_bos(screen
, time
);
332 static struct v3d_bo
*
333 v3d_bo_open_handle(struct v3d_screen
*screen
,
334 uint32_t handle
, uint32_t size
)
340 mtx_lock(&screen
->bo_handles_mutex
);
342 bo
= util_hash_table_get(screen
->bo_handles
, (void*)(uintptr_t)handle
);
344 pipe_reference(NULL
, &bo
->reference
);
348 bo
= CALLOC_STRUCT(v3d_bo
);
349 pipe_reference_init(&bo
->reference
, 1);
356 #ifdef USE_V3D_SIMULATOR
357 v3d_simulator_open_from_handle(screen
->fd
, bo
->handle
, bo
->size
);
358 bo
->map
= malloc(bo
->size
);
361 struct drm_v3d_get_bo_offset get
= {
364 int ret
= v3d_ioctl(screen
->fd
, DRM_IOCTL_V3D_GET_BO_OFFSET
, &get
);
366 fprintf(stderr
, "Failed to get BO offset: %s\n",
372 bo
->offset
= get
.offset
;
373 assert(bo
->offset
!= 0);
375 util_hash_table_set(screen
->bo_handles
, (void *)(uintptr_t)handle
, bo
);
378 mtx_unlock(&screen
->bo_handles_mutex
);
383 v3d_bo_open_name(struct v3d_screen
*screen
, uint32_t name
)
385 struct drm_gem_open o
= {
388 int ret
= v3d_ioctl(screen
->fd
, DRM_IOCTL_GEM_OPEN
, &o
);
390 fprintf(stderr
, "Failed to open bo %d: %s\n",
391 name
, strerror(errno
));
395 return v3d_bo_open_handle(screen
, o
.handle
, o
.size
);
399 v3d_bo_open_dmabuf(struct v3d_screen
*screen
, int fd
)
402 int ret
= drmPrimeFDToHandle(screen
->fd
, fd
, &handle
);
405 fprintf(stderr
, "Failed to get v3d handle for dmabuf %d\n", fd
);
409 /* Determine the size of the bo we were handed. */
410 size
= lseek(fd
, 0, SEEK_END
);
412 fprintf(stderr
, "Couldn't get size of dmabuf fd %d.\n", fd
);
416 return v3d_bo_open_handle(screen
, handle
, size
);
420 v3d_bo_get_dmabuf(struct v3d_bo
*bo
)
423 int ret
= drmPrimeHandleToFD(bo
->screen
->fd
, bo
->handle
,
426 fprintf(stderr
, "Failed to export gem bo %d to dmabuf\n",
431 mtx_lock(&bo
->screen
->bo_handles_mutex
);
433 util_hash_table_set(bo
->screen
->bo_handles
, (void *)(uintptr_t)bo
->handle
, bo
);
434 mtx_unlock(&bo
->screen
->bo_handles_mutex
);
440 v3d_bo_flink(struct v3d_bo
*bo
, uint32_t *name
)
442 struct drm_gem_flink flink
= {
443 .handle
= bo
->handle
,
445 int ret
= v3d_ioctl(bo
->screen
->fd
, DRM_IOCTL_GEM_FLINK
, &flink
);
447 fprintf(stderr
, "Failed to flink bo %d: %s\n",
448 bo
->handle
, strerror(errno
));
459 static int v3d_wait_bo_ioctl(int fd
, uint32_t handle
, uint64_t timeout_ns
)
461 struct drm_v3d_wait_bo wait
= {
463 .timeout_ns
= timeout_ns
,
465 int ret
= v3d_ioctl(fd
, DRM_IOCTL_V3D_WAIT_BO
, &wait
);
474 v3d_bo_wait(struct v3d_bo
*bo
, uint64_t timeout_ns
, const char *reason
)
476 struct v3d_screen
*screen
= bo
->screen
;
478 if (unlikely(V3D_DEBUG
& V3D_DEBUG_PERF
) && timeout_ns
&& reason
) {
479 if (v3d_wait_bo_ioctl(screen
->fd
, bo
->handle
, 0) == -ETIME
) {
480 fprintf(stderr
, "Blocking on %s BO for %s\n",
485 int ret
= v3d_wait_bo_ioctl(screen
->fd
, bo
->handle
, timeout_ns
);
488 fprintf(stderr
, "wait failed: %d\n", ret
);
499 v3d_bo_map_unsynchronized(struct v3d_bo
*bo
)
507 struct drm_v3d_mmap_bo map
;
508 memset(&map
, 0, sizeof(map
));
509 map
.handle
= bo
->handle
;
510 ret
= v3d_ioctl(bo
->screen
->fd
, DRM_IOCTL_V3D_MMAP_BO
, &map
);
513 fprintf(stderr
, "map ioctl failure\n");
517 bo
->map
= mmap(NULL
, bo
->size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
518 bo
->screen
->fd
, offset
);
519 if (bo
->map
== MAP_FAILED
) {
520 fprintf(stderr
, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
521 bo
->handle
, (long long)offset
, bo
->size
);
524 VG(VALGRIND_MALLOCLIKE_BLOCK(bo
->map
, bo
->size
, 0, false));
530 v3d_bo_map(struct v3d_bo
*bo
)
532 void *map
= v3d_bo_map_unsynchronized(bo
);
534 bool ok
= v3d_bo_wait(bo
, PIPE_TIMEOUT_INFINITE
, "bo map");
536 fprintf(stderr
, "BO wait for map failed\n");
544 v3d_bufmgr_destroy(struct pipe_screen
*pscreen
)
546 struct v3d_screen
*screen
= v3d_screen(pscreen
);
547 struct v3d_bo_cache
*cache
= &screen
->bo_cache
;
549 v3d_bo_cache_free_all(cache
);
552 fprintf(stderr
, "BO stats after screen destroy:\n");
553 v3d_bo_dump_stats(screen
);