2 * Copyright (C) 2017-2019 Lima Project
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
25 #include <sys/types.h>
30 #include "drm-uapi/lima_drm.h"
32 #include "util/u_hash_table.h"
33 #include "util/u_math.h"
34 #include "util/os_time.h"
35 #include "os/os_mman.h"
37 #include "frontend/drm_driver.h"
39 #include "lima_screen.h"
41 #include "lima_util.h"
43 bool lima_bo_table_init(struct lima_screen
*screen
)
45 screen
->bo_handles
= util_hash_table_create_ptr_keys();
46 if (!screen
->bo_handles
)
49 screen
->bo_flink_names
= util_hash_table_create_ptr_keys();
50 if (!screen
->bo_flink_names
)
53 mtx_init(&screen
->bo_table_lock
, mtx_plain
);
57 _mesa_hash_table_destroy(screen
->bo_handles
, NULL
);
61 bool lima_bo_cache_init(struct lima_screen
*screen
)
63 mtx_init(&screen
->bo_cache_lock
, mtx_plain
);
64 list_inithead(&screen
->bo_cache_time
);
65 for (int i
= 0; i
< NR_BO_CACHE_BUCKETS
; i
++)
66 list_inithead(&screen
->bo_cache_buckets
[i
]);
71 void lima_bo_table_fini(struct lima_screen
*screen
)
73 mtx_destroy(&screen
->bo_table_lock
);
74 _mesa_hash_table_destroy(screen
->bo_handles
, NULL
);
75 _mesa_hash_table_destroy(screen
->bo_flink_names
, NULL
);
79 lima_bo_cache_remove(struct lima_bo
*bo
)
81 list_del(&bo
->size_list
);
82 list_del(&bo
->time_list
);
85 static void lima_close_kms_handle(struct lima_screen
*screen
, uint32_t handle
)
87 struct drm_gem_close args
= {
91 drmIoctl(screen
->fd
, DRM_IOCTL_GEM_CLOSE
, &args
);
95 lima_bo_free(struct lima_bo
*bo
)
97 struct lima_screen
*screen
= bo
->screen
;
99 if (lima_debug
& LIMA_DEBUG_BO_CACHE
)
100 fprintf(stderr
, "%s: %p (size=%d)\n", __func__
,
103 mtx_lock(&screen
->bo_table_lock
);
104 _mesa_hash_table_remove_key(screen
->bo_handles
,
105 (void *)(uintptr_t)bo
->handle
);
107 _mesa_hash_table_remove_key(screen
->bo_flink_names
,
108 (void *)(uintptr_t)bo
->flink_name
);
109 mtx_unlock(&screen
->bo_table_lock
);
114 lima_close_kms_handle(screen
, bo
->handle
);
118 void lima_bo_cache_fini(struct lima_screen
*screen
)
120 mtx_destroy(&screen
->bo_cache_lock
);
122 list_for_each_entry_safe(struct lima_bo
, entry
,
123 &screen
->bo_cache_time
, time_list
) {
124 lima_bo_cache_remove(entry
);
129 static bool lima_bo_get_info(struct lima_bo
*bo
)
131 struct drm_lima_gem_info req
= {
132 .handle
= bo
->handle
,
135 if(drmIoctl(bo
->screen
->fd
, DRM_IOCTL_LIMA_GEM_INFO
, &req
))
138 bo
->offset
= req
.offset
;
144 lima_bucket_index(unsigned size
)
146 /* Round down to POT to compute a bucket index */
148 unsigned bucket_index
= util_logbase2(size
);
150 /* Clamp the bucket index; all huge allocations will be
151 * sorted into the largest bucket */
152 bucket_index
= CLAMP(bucket_index
, MIN_BO_CACHE_BUCKET
,
153 MAX_BO_CACHE_BUCKET
);
156 return (bucket_index
- MIN_BO_CACHE_BUCKET
);
159 static struct list_head
*
160 lima_bo_cache_get_bucket(struct lima_screen
*screen
, unsigned size
)
162 return &screen
->bo_cache_buckets
[lima_bucket_index(size
)];
166 lima_bo_cache_free_stale_bos(struct lima_screen
*screen
, time_t time
)
169 list_for_each_entry_safe(struct lima_bo
, entry
,
170 &screen
->bo_cache_time
, time_list
) {
171 /* Free BOs that are sitting idle for longer than 5 seconds */
172 if (time
- entry
->free_time
> 6) {
173 lima_bo_cache_remove(entry
);
179 if ((lima_debug
& LIMA_DEBUG_BO_CACHE
) && cnt
)
180 fprintf(stderr
, "%s: freed %d stale BOs\n", __func__
, cnt
);
184 lima_bo_cache_print_stats(struct lima_screen
*screen
)
186 fprintf(stderr
, "===============\n");
187 fprintf(stderr
, "BO cache stats:\n");
188 unsigned total_size
= 0;
189 for (int i
= 0; i
< NR_BO_CACHE_BUCKETS
; i
++) {
190 struct list_head
*bucket
= &screen
->bo_cache_buckets
[i
];
191 unsigned bucket_size
= 0;
192 list_for_each_entry(struct lima_bo
, entry
, bucket
, size_list
) {
193 bucket_size
+= entry
->size
;
194 total_size
+= entry
->size
;
196 fprintf(stderr
, "Bucket #%d, BOs: %d, size: %u\n", i
,
200 fprintf(stderr
, "Total size: %u\n", total_size
);
204 lima_bo_cache_put(struct lima_bo
*bo
)
209 struct lima_screen
*screen
= bo
->screen
;
211 mtx_lock(&screen
->bo_cache_lock
);
212 struct list_head
*bucket
= lima_bo_cache_get_bucket(screen
, bo
->size
);
215 mtx_unlock(&screen
->bo_cache_lock
);
219 struct timespec time
;
220 clock_gettime(CLOCK_MONOTONIC
, &time
);
221 bo
->free_time
= time
.tv_sec
;
222 list_addtail(&bo
->size_list
, bucket
);
223 list_addtail(&bo
->time_list
, &screen
->bo_cache_time
);
224 lima_bo_cache_free_stale_bos(screen
, time
.tv_sec
);
225 if (lima_debug
& LIMA_DEBUG_BO_CACHE
) {
226 fprintf(stderr
, "%s: put BO: %p (size=%d)\n", __func__
, bo
, bo
->size
);
227 lima_bo_cache_print_stats(screen
);
229 mtx_unlock(&screen
->bo_cache_lock
);
234 static struct lima_bo
*
235 lima_bo_cache_get(struct lima_screen
*screen
, uint32_t size
, uint32_t flags
)
237 /* we won't cache heap buffer */
238 if (flags
& LIMA_BO_FLAG_HEAP
)
241 struct lima_bo
*bo
= NULL
;
242 mtx_lock(&screen
->bo_cache_lock
);
243 struct list_head
*bucket
= lima_bo_cache_get_bucket(screen
, size
);
246 mtx_unlock(&screen
->bo_cache_lock
);
250 list_for_each_entry_safe(struct lima_bo
, entry
, bucket
, size_list
) {
251 if (entry
->size
>= size
) {
252 /* Check if BO is idle. If it's not it's better to allocate new one */
253 if (!lima_bo_wait(entry
, LIMA_GEM_WAIT_WRITE
, 0)) {
254 if (lima_debug
& LIMA_DEBUG_BO_CACHE
) {
255 fprintf(stderr
, "%s: found BO %p but it's busy\n", __func__
,
261 lima_bo_cache_remove(entry
);
262 p_atomic_set(&entry
->refcnt
, 1);
263 entry
->flags
= flags
;
265 if (lima_debug
& LIMA_DEBUG_BO_CACHE
) {
266 fprintf(stderr
, "%s: got BO: %p (size=%d), requested size %d\n",
267 __func__
, bo
, bo
->size
, size
);
268 lima_bo_cache_print_stats(screen
);
274 mtx_unlock(&screen
->bo_cache_lock
);
279 struct lima_bo
*lima_bo_create(struct lima_screen
*screen
,
280 uint32_t size
, uint32_t flags
)
284 size
= align(size
, LIMA_PAGE_SIZE
);
286 /* Try to get bo from cache first */
287 bo
= lima_bo_cache_get(screen
, size
, flags
);
291 struct drm_lima_gem_create req
= {
296 if (!(bo
= calloc(1, sizeof(*bo
))))
299 list_inithead(&bo
->time_list
);
300 list_inithead(&bo
->size_list
);
302 if (drmIoctl(screen
->fd
, DRM_IOCTL_LIMA_GEM_CREATE
, &req
))
307 bo
->flags
= req
.flags
;
308 bo
->handle
= req
.handle
;
309 bo
->cacheable
= !(lima_debug
& LIMA_DEBUG_NO_BO_CACHE
||
310 flags
& LIMA_BO_FLAG_HEAP
);
311 p_atomic_set(&bo
->refcnt
, 1);
313 if (!lima_bo_get_info(bo
))
316 if (lima_debug
& LIMA_DEBUG_BO_CACHE
)
317 fprintf(stderr
, "%s: %p (size=%d)\n", __func__
,
323 lima_close_kms_handle(screen
, bo
->handle
);
329 void lima_bo_unreference(struct lima_bo
*bo
)
331 if (!p_atomic_dec_zero(&bo
->refcnt
))
334 /* Try to put it into cache */
335 if (lima_bo_cache_put(bo
))
341 void *lima_bo_map(struct lima_bo
*bo
)
344 bo
->map
= os_mmap(0, bo
->size
, PROT_READ
| PROT_WRITE
,
345 MAP_SHARED
, bo
->screen
->fd
, bo
->offset
);
346 if (bo
->map
== MAP_FAILED
)
353 void lima_bo_unmap(struct lima_bo
*bo
)
356 os_munmap(bo
->map
, bo
->size
);
361 bool lima_bo_export(struct lima_bo
*bo
, struct winsys_handle
*handle
)
363 struct lima_screen
*screen
= bo
->screen
;
365 /* Don't cache exported BOs */
366 bo
->cacheable
= false;
368 switch (handle
->type
) {
369 case WINSYS_HANDLE_TYPE_SHARED
:
370 if (!bo
->flink_name
) {
371 struct drm_gem_flink flink
= {
372 .handle
= bo
->handle
,
375 if (drmIoctl(screen
->fd
, DRM_IOCTL_GEM_FLINK
, &flink
))
378 bo
->flink_name
= flink
.name
;
380 mtx_lock(&screen
->bo_table_lock
);
381 _mesa_hash_table_insert(screen
->bo_flink_names
,
382 (void *)(uintptr_t)bo
->flink_name
, bo
);
383 mtx_unlock(&screen
->bo_table_lock
);
385 handle
->handle
= bo
->flink_name
;
388 case WINSYS_HANDLE_TYPE_KMS
:
389 mtx_lock(&screen
->bo_table_lock
);
390 _mesa_hash_table_insert(screen
->bo_handles
,
391 (void *)(uintptr_t)bo
->handle
, bo
);
392 mtx_unlock(&screen
->bo_table_lock
);
394 handle
->handle
= bo
->handle
;
397 case WINSYS_HANDLE_TYPE_FD
:
398 if (drmPrimeHandleToFD(screen
->fd
, bo
->handle
, DRM_CLOEXEC
,
399 (int*)&handle
->handle
))
402 mtx_lock(&screen
->bo_table_lock
);
403 _mesa_hash_table_insert(screen
->bo_handles
,
404 (void *)(uintptr_t)bo
->handle
, bo
);
405 mtx_unlock(&screen
->bo_table_lock
);
413 struct lima_bo
*lima_bo_import(struct lima_screen
*screen
,
414 struct winsys_handle
*handle
)
416 struct lima_bo
*bo
= NULL
;
417 struct drm_gem_open req
= {0};
418 uint32_t dma_buf_size
= 0;
419 unsigned h
= handle
->handle
;
421 mtx_lock(&screen
->bo_table_lock
);
423 /* Convert a DMA buf handle to a KMS handle now. */
424 if (handle
->type
== WINSYS_HANDLE_TYPE_FD
) {
425 uint32_t prime_handle
;
428 /* Get a KMS handle. */
429 if (drmPrimeFDToHandle(screen
->fd
, h
, &prime_handle
)) {
430 mtx_unlock(&screen
->bo_table_lock
);
434 /* Query the buffer size. */
435 size
= lseek(h
, 0, SEEK_END
);
436 if (size
== (off_t
)-1) {
437 mtx_unlock(&screen
->bo_table_lock
);
438 lima_close_kms_handle(screen
, prime_handle
);
441 lseek(h
, 0, SEEK_SET
);
447 switch (handle
->type
) {
448 case WINSYS_HANDLE_TYPE_SHARED
:
449 bo
= util_hash_table_get(screen
->bo_flink_names
,
450 (void *)(uintptr_t)h
);
452 case WINSYS_HANDLE_TYPE_KMS
:
453 case WINSYS_HANDLE_TYPE_FD
:
454 bo
= util_hash_table_get(screen
->bo_handles
,
455 (void *)(uintptr_t)h
);
458 mtx_unlock(&screen
->bo_table_lock
);
463 p_atomic_inc(&bo
->refcnt
);
464 /* Don't cache imported BOs */
465 bo
->cacheable
= false;
466 mtx_unlock(&screen
->bo_table_lock
);
470 if (!(bo
= calloc(1, sizeof(*bo
)))) {
471 mtx_unlock(&screen
->bo_table_lock
);
472 if (handle
->type
== WINSYS_HANDLE_TYPE_FD
)
473 lima_close_kms_handle(screen
, h
);
477 /* Don't cache imported BOs */
478 bo
->cacheable
= false;
479 list_inithead(&bo
->time_list
);
480 list_inithead(&bo
->size_list
);
482 p_atomic_set(&bo
->refcnt
, 1);
484 switch (handle
->type
) {
485 case WINSYS_HANDLE_TYPE_SHARED
:
487 if (drmIoctl(screen
->fd
, DRM_IOCTL_GEM_OPEN
, &req
)) {
488 mtx_unlock(&screen
->bo_table_lock
);
492 bo
->handle
= req
.handle
;
496 case WINSYS_HANDLE_TYPE_FD
:
498 bo
->size
= dma_buf_size
;
505 if (lima_bo_get_info(bo
)) {
506 if (handle
->type
== WINSYS_HANDLE_TYPE_SHARED
)
507 _mesa_hash_table_insert(screen
->bo_flink_names
,
508 (void *)(uintptr_t)bo
->flink_name
, bo
);
509 _mesa_hash_table_insert(screen
->bo_handles
,
510 (void*)(uintptr_t)bo
->handle
, bo
);
513 lima_close_kms_handle(screen
, bo
->handle
);
518 mtx_unlock(&screen
->bo_table_lock
);
523 bool lima_bo_wait(struct lima_bo
*bo
, uint32_t op
, uint64_t timeout_ns
)
530 abs_timeout
= os_time_get_absolute_timeout(timeout_ns
);
532 if (abs_timeout
== OS_TIMEOUT_INFINITE
)
533 abs_timeout
= INT64_MAX
;
535 struct drm_lima_gem_wait req
= {
536 .handle
= bo
->handle
,
538 .timeout_ns
= abs_timeout
,
541 return drmIoctl(bo
->screen
->fd
, DRM_IOCTL_LIMA_GEM_WAIT
, &req
) == 0;