2 * Copyright (C) 2017-2019 Lima Project
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
25 #include <sys/types.h>
30 #include "drm-uapi/lima_drm.h"
32 #include "util/u_hash_table.h"
33 #include "util/u_math.h"
34 #include "util/os_time.h"
35 #include "os/os_mman.h"
37 #include "state_tracker/drm_driver.h"
39 #include "lima_screen.h"
41 #include "lima_util.h"
43 #define PTR_TO_UINT(x) ((unsigned)((intptr_t)(x)))
45 static unsigned handle_hash(void *key
)
47 return PTR_TO_UINT(key
);
50 static int handle_compare(void *key1
, void *key2
)
52 return PTR_TO_UINT(key1
) != PTR_TO_UINT(key2
);
55 bool lima_bo_table_init(struct lima_screen
*screen
)
57 screen
->bo_handles
= util_hash_table_create(handle_hash
, handle_compare
);
58 if (!screen
->bo_handles
)
61 screen
->bo_flink_names
= util_hash_table_create(handle_hash
, handle_compare
);
62 if (!screen
->bo_flink_names
)
65 mtx_init(&screen
->bo_table_lock
, mtx_plain
);
69 util_hash_table_destroy(screen
->bo_handles
);
73 bool lima_bo_cache_init(struct lima_screen
*screen
)
75 mtx_init(&screen
->bo_cache_lock
, mtx_plain
);
76 list_inithead(&screen
->bo_cache_time
);
77 for (int i
= 0; i
< NR_BO_CACHE_BUCKETS
; i
++)
78 list_inithead(&screen
->bo_cache_buckets
[i
]);
83 void lima_bo_table_fini(struct lima_screen
*screen
)
85 mtx_destroy(&screen
->bo_table_lock
);
86 util_hash_table_destroy(screen
->bo_handles
);
87 util_hash_table_destroy(screen
->bo_flink_names
);
91 lima_bo_cache_remove(struct lima_bo
*bo
)
93 list_del(&bo
->size_list
);
94 list_del(&bo
->time_list
);
97 static void lima_close_kms_handle(struct lima_screen
*screen
, uint32_t handle
)
99 struct drm_gem_close args
= {
103 drmIoctl(screen
->fd
, DRM_IOCTL_GEM_CLOSE
, &args
);
107 lima_bo_free(struct lima_bo
*bo
)
109 struct lima_screen
*screen
= bo
->screen
;
111 if (lima_debug
& LIMA_DEBUG_BO_CACHE
)
112 fprintf(stderr
, "%s: %p (size=%d)\n", __func__
,
115 mtx_lock(&screen
->bo_table_lock
);
116 util_hash_table_remove(screen
->bo_handles
,
117 (void *)(uintptr_t)bo
->handle
);
119 util_hash_table_remove(screen
->bo_flink_names
,
120 (void *)(uintptr_t)bo
->flink_name
);
121 mtx_unlock(&screen
->bo_table_lock
);
126 lima_close_kms_handle(screen
, bo
->handle
);
130 void lima_bo_cache_fini(struct lima_screen
*screen
)
132 mtx_destroy(&screen
->bo_cache_lock
);
134 list_for_each_entry_safe(struct lima_bo
, entry
,
135 &screen
->bo_cache_time
, time_list
) {
136 lima_bo_cache_remove(entry
);
141 static bool lima_bo_get_info(struct lima_bo
*bo
)
143 struct drm_lima_gem_info req
= {
144 .handle
= bo
->handle
,
147 if(drmIoctl(bo
->screen
->fd
, DRM_IOCTL_LIMA_GEM_INFO
, &req
))
150 bo
->offset
= req
.offset
;
156 lima_bucket_index(unsigned size
)
158 /* Round down to POT to compute a bucket index */
160 unsigned bucket_index
= util_logbase2(size
);
162 /* Clamp the bucket index; all huge allocations will be
163 * sorted into the largest bucket */
164 bucket_index
= CLAMP(bucket_index
, MIN_BO_CACHE_BUCKET
,
165 MAX_BO_CACHE_BUCKET
);
168 return (bucket_index
- MIN_BO_CACHE_BUCKET
);
171 static struct list_head
*
172 lima_bo_cache_get_bucket(struct lima_screen
*screen
, unsigned size
)
174 return &screen
->bo_cache_buckets
[lima_bucket_index(size
)];
178 lima_bo_cache_free_stale_bos(struct lima_screen
*screen
, time_t time
)
181 list_for_each_entry_safe(struct lima_bo
, entry
,
182 &screen
->bo_cache_time
, time_list
) {
183 /* Free BOs that are sitting idle for longer than 5 seconds */
184 if (time
- entry
->free_time
> 6) {
185 lima_bo_cache_remove(entry
);
191 if ((lima_debug
& LIMA_DEBUG_BO_CACHE
) && cnt
)
192 fprintf(stderr
, "%s: freed %d stale BOs\n", __func__
, cnt
);
196 lima_bo_cache_print_stats(struct lima_screen
*screen
)
198 fprintf(stderr
, "===============\n");
199 fprintf(stderr
, "BO cache stats:\n");
200 unsigned total_size
= 0;
201 for (int i
= 0; i
< NR_BO_CACHE_BUCKETS
; i
++) {
202 struct list_head
*bucket
= &screen
->bo_cache_buckets
[i
];
203 unsigned bucket_size
= 0;
204 list_for_each_entry(struct lima_bo
, entry
, bucket
, size_list
) {
205 bucket_size
+= entry
->size
;
206 total_size
+= entry
->size
;
208 fprintf(stderr
, "Bucket #%d, BOs: %d, size: %u\n", i
,
212 fprintf(stderr
, "Total size: %u\n", total_size
);
216 lima_bo_cache_put(struct lima_bo
*bo
)
221 struct lima_screen
*screen
= bo
->screen
;
223 mtx_lock(&screen
->bo_cache_lock
);
224 struct list_head
*bucket
= lima_bo_cache_get_bucket(screen
, bo
->size
);
227 mtx_unlock(&screen
->bo_cache_lock
);
231 struct timespec time
;
232 clock_gettime(CLOCK_MONOTONIC
, &time
);
233 bo
->free_time
= time
.tv_sec
;
234 list_addtail(&bo
->size_list
, bucket
);
235 list_addtail(&bo
->time_list
, &screen
->bo_cache_time
);
236 lima_bo_cache_free_stale_bos(screen
, time
.tv_sec
);
237 if (lima_debug
& LIMA_DEBUG_BO_CACHE
) {
238 fprintf(stderr
, "%s: put BO: %p (size=%d)\n", __func__
, bo
, bo
->size
);
239 lima_bo_cache_print_stats(screen
);
241 mtx_unlock(&screen
->bo_cache_lock
);
246 static struct lima_bo
*
247 lima_bo_cache_get(struct lima_screen
*screen
, uint32_t size
, uint32_t flags
)
249 struct lima_bo
*bo
= NULL
;
250 mtx_lock(&screen
->bo_cache_lock
);
251 struct list_head
*bucket
= lima_bo_cache_get_bucket(screen
, size
);
254 mtx_unlock(&screen
->bo_cache_lock
);
258 list_for_each_entry_safe(struct lima_bo
, entry
, bucket
, size_list
) {
259 if (entry
->size
>= size
) {
260 /* Check if BO is idle. If it's not it's better to allocate new one */
261 if (!lima_bo_wait(entry
, LIMA_GEM_WAIT_WRITE
, 0)) {
262 if (lima_debug
& LIMA_DEBUG_BO_CACHE
) {
263 fprintf(stderr
, "%s: found BO %p but it's busy\n", __func__
,
269 lima_bo_cache_remove(entry
);
270 p_atomic_set(&entry
->refcnt
, 1);
271 entry
->flags
= flags
;
273 if (lima_debug
& LIMA_DEBUG_BO_CACHE
) {
274 fprintf(stderr
, "%s: got BO: %p (size=%d), requested size %d\n",
275 __func__
, bo
, bo
->size
, size
);
276 lima_bo_cache_print_stats(screen
);
282 mtx_unlock(&screen
->bo_cache_lock
);
287 struct lima_bo
*lima_bo_create(struct lima_screen
*screen
,
288 uint32_t size
, uint32_t flags
)
292 size
= align(size
, LIMA_PAGE_SIZE
);
294 /* Try to get bo from cache first */
295 bo
= lima_bo_cache_get(screen
, size
, flags
);
299 struct drm_lima_gem_create req
= {
304 if (!(bo
= calloc(1, sizeof(*bo
))))
307 list_inithead(&bo
->time_list
);
308 list_inithead(&bo
->size_list
);
310 if (drmIoctl(screen
->fd
, DRM_IOCTL_LIMA_GEM_CREATE
, &req
))
315 bo
->flags
= req
.flags
;
316 bo
->handle
= req
.handle
;
317 bo
->cacheable
= !(lima_debug
& LIMA_DEBUG_NO_BO_CACHE
);
318 p_atomic_set(&bo
->refcnt
, 1);
320 if (!lima_bo_get_info(bo
))
323 if (lima_debug
& LIMA_DEBUG_BO_CACHE
)
324 fprintf(stderr
, "%s: %p (size=%d)\n", __func__
,
330 lima_close_kms_handle(screen
, bo
->handle
);
336 void lima_bo_unreference(struct lima_bo
*bo
)
338 if (!p_atomic_dec_zero(&bo
->refcnt
))
341 /* Try to put it into cache */
342 if (lima_bo_cache_put(bo
))
348 void *lima_bo_map(struct lima_bo
*bo
)
351 bo
->map
= os_mmap(0, bo
->size
, PROT_READ
| PROT_WRITE
,
352 MAP_SHARED
, bo
->screen
->fd
, bo
->offset
);
353 if (bo
->map
== MAP_FAILED
)
360 void lima_bo_unmap(struct lima_bo
*bo
)
363 os_munmap(bo
->map
, bo
->size
);
368 bool lima_bo_export(struct lima_bo
*bo
, struct winsys_handle
*handle
)
370 struct lima_screen
*screen
= bo
->screen
;
372 /* Don't cache exported BOs */
373 bo
->cacheable
= false;
375 switch (handle
->type
) {
376 case WINSYS_HANDLE_TYPE_SHARED
:
377 if (!bo
->flink_name
) {
378 struct drm_gem_flink flink
= {
379 .handle
= bo
->handle
,
382 if (drmIoctl(screen
->fd
, DRM_IOCTL_GEM_FLINK
, &flink
))
385 bo
->flink_name
= flink
.name
;
387 mtx_lock(&screen
->bo_table_lock
);
388 util_hash_table_set(screen
->bo_flink_names
,
389 (void *)(uintptr_t)bo
->flink_name
, bo
);
390 mtx_unlock(&screen
->bo_table_lock
);
392 handle
->handle
= bo
->flink_name
;
395 case WINSYS_HANDLE_TYPE_KMS
:
396 mtx_lock(&screen
->bo_table_lock
);
397 util_hash_table_set(screen
->bo_handles
,
398 (void *)(uintptr_t)bo
->handle
, bo
);
399 mtx_unlock(&screen
->bo_table_lock
);
401 handle
->handle
= bo
->handle
;
404 case WINSYS_HANDLE_TYPE_FD
:
405 if (drmPrimeHandleToFD(screen
->fd
, bo
->handle
, DRM_CLOEXEC
,
406 (int*)&handle
->handle
))
409 mtx_lock(&screen
->bo_table_lock
);
410 util_hash_table_set(screen
->bo_handles
,
411 (void *)(uintptr_t)bo
->handle
, bo
);
412 mtx_unlock(&screen
->bo_table_lock
);
420 struct lima_bo
*lima_bo_import(struct lima_screen
*screen
,
421 struct winsys_handle
*handle
)
423 struct lima_bo
*bo
= NULL
;
424 struct drm_gem_open req
= {0};
425 uint32_t dma_buf_size
= 0;
426 unsigned h
= handle
->handle
;
428 mtx_lock(&screen
->bo_table_lock
);
430 /* Convert a DMA buf handle to a KMS handle now. */
431 if (handle
->type
== WINSYS_HANDLE_TYPE_FD
) {
432 uint32_t prime_handle
;
435 /* Get a KMS handle. */
436 if (drmPrimeFDToHandle(screen
->fd
, h
, &prime_handle
)) {
437 mtx_unlock(&screen
->bo_table_lock
);
441 /* Query the buffer size. */
442 size
= lseek(h
, 0, SEEK_END
);
443 if (size
== (off_t
)-1) {
444 mtx_unlock(&screen
->bo_table_lock
);
445 lima_close_kms_handle(screen
, prime_handle
);
448 lseek(h
, 0, SEEK_SET
);
454 switch (handle
->type
) {
455 case WINSYS_HANDLE_TYPE_SHARED
:
456 bo
= util_hash_table_get(screen
->bo_flink_names
,
457 (void *)(uintptr_t)h
);
459 case WINSYS_HANDLE_TYPE_KMS
:
460 case WINSYS_HANDLE_TYPE_FD
:
461 bo
= util_hash_table_get(screen
->bo_handles
,
462 (void *)(uintptr_t)h
);
465 mtx_unlock(&screen
->bo_table_lock
);
470 p_atomic_inc(&bo
->refcnt
);
471 /* Don't cache imported BOs */
472 bo
->cacheable
= false;
473 mtx_unlock(&screen
->bo_table_lock
);
477 if (!(bo
= calloc(1, sizeof(*bo
)))) {
478 mtx_unlock(&screen
->bo_table_lock
);
479 if (handle
->type
== WINSYS_HANDLE_TYPE_FD
)
480 lima_close_kms_handle(screen
, h
);
484 /* Don't cache imported BOs */
485 bo
->cacheable
= false;
486 list_inithead(&bo
->time_list
);
487 list_inithead(&bo
->size_list
);
489 p_atomic_set(&bo
->refcnt
, 1);
491 switch (handle
->type
) {
492 case WINSYS_HANDLE_TYPE_SHARED
:
494 if (drmIoctl(screen
->fd
, DRM_IOCTL_GEM_OPEN
, &req
)) {
495 mtx_unlock(&screen
->bo_table_lock
);
499 bo
->handle
= req
.handle
;
503 case WINSYS_HANDLE_TYPE_FD
:
505 bo
->size
= dma_buf_size
;
512 if (lima_bo_get_info(bo
)) {
513 if (handle
->type
== WINSYS_HANDLE_TYPE_SHARED
)
514 util_hash_table_set(screen
->bo_flink_names
,
515 (void *)(uintptr_t)bo
->flink_name
, bo
);
516 util_hash_table_set(screen
->bo_handles
,
517 (void*)(uintptr_t)bo
->handle
, bo
);
520 lima_close_kms_handle(screen
, bo
->handle
);
525 mtx_unlock(&screen
->bo_table_lock
);
530 bool lima_bo_wait(struct lima_bo
*bo
, uint32_t op
, uint64_t timeout_ns
)
537 abs_timeout
= os_time_get_absolute_timeout(timeout_ns
);
539 if (abs_timeout
== OS_TIMEOUT_INFINITE
)
540 abs_timeout
= INT64_MAX
;
542 struct drm_lima_gem_wait req
= {
543 .handle
= bo
->handle
,
545 .timeout_ns
= abs_timeout
,
548 return drmIoctl(bo
->screen
->fd
, DRM_IOCTL_LIMA_GEM_WAIT
, &req
) == 0;