e359f1aa41d807e75c67c79b18545c526ef89b37
[mesa.git] / src / gallium / winsys / amdgpu / drm / amdgpu_bo.c
1 /*
2 * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
3 * Copyright © 2015 Advanced Micro Devices, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
16 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
18 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * The above copyright notice and this permission notice (including the
24 * next paragraph) shall be included in all copies or substantial portions
25 * of the Software.
26 */
27
28 #include "amdgpu_cs.h"
29
30 #include "util/hash_table.h"
31 #include "util/os_time.h"
32 #include "util/u_hash_table.h"
33 #include "frontend/drm_driver.h"
34 #include "drm-uapi/amdgpu_drm.h"
35 #include <xf86drm.h>
36 #include <stdio.h>
37 #include <inttypes.h>
38
39 #ifndef AMDGPU_VA_RANGE_HIGH
40 #define AMDGPU_VA_RANGE_HIGH 0x2
41 #endif
42
43 /* Set to 1 for verbose output showing committed sparse buffer ranges. */
44 #define DEBUG_SPARSE_COMMITS 0
45
46 struct amdgpu_sparse_backing_chunk {
47 uint32_t begin, end;
48 };
49
50 static void amdgpu_bo_unmap(struct pb_buffer *buf);
51
52 static bool amdgpu_bo_wait(struct pb_buffer *_buf, uint64_t timeout,
53 enum radeon_bo_usage usage)
54 {
55 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
56 struct amdgpu_winsys *ws = bo->ws;
57 int64_t abs_timeout;
58
59 if (timeout == 0) {
60 if (p_atomic_read(&bo->num_active_ioctls))
61 return false;
62
63 } else {
64 abs_timeout = os_time_get_absolute_timeout(timeout);
65
66 /* Wait if any ioctl is being submitted with this buffer. */
67 if (!os_wait_until_zero_abs_timeout(&bo->num_active_ioctls, abs_timeout))
68 return false;
69 }
70
71 if (bo->is_shared) {
72 /* We can't use user fences for shared buffers, because user fences
73 * are local to this process only. If we want to wait for all buffer
74 * uses in all processes, we have to use amdgpu_bo_wait_for_idle.
75 */
76 bool buffer_busy = true;
77 int r;
78
79 r = amdgpu_bo_wait_for_idle(bo->bo, timeout, &buffer_busy);
80 if (r)
81 fprintf(stderr, "%s: amdgpu_bo_wait_for_idle failed %i\n", __func__,
82 r);
83 return !buffer_busy;
84 }
85
86 if (timeout == 0) {
87 unsigned idle_fences;
88 bool buffer_idle;
89
90 simple_mtx_lock(&ws->bo_fence_lock);
91
92 for (idle_fences = 0; idle_fences < bo->num_fences; ++idle_fences) {
93 if (!amdgpu_fence_wait(bo->fences[idle_fences], 0, false))
94 break;
95 }
96
97 /* Release the idle fences to avoid checking them again later. */
98 for (unsigned i = 0; i < idle_fences; ++i)
99 amdgpu_fence_reference(&bo->fences[i], NULL);
100
101 memmove(&bo->fences[0], &bo->fences[idle_fences],
102 (bo->num_fences - idle_fences) * sizeof(*bo->fences));
103 bo->num_fences -= idle_fences;
104
105 buffer_idle = !bo->num_fences;
106 simple_mtx_unlock(&ws->bo_fence_lock);
107
108 return buffer_idle;
109 } else {
110 bool buffer_idle = true;
111
112 simple_mtx_lock(&ws->bo_fence_lock);
113 while (bo->num_fences && buffer_idle) {
114 struct pipe_fence_handle *fence = NULL;
115 bool fence_idle = false;
116
117 amdgpu_fence_reference(&fence, bo->fences[0]);
118
119 /* Wait for the fence. */
120 simple_mtx_unlock(&ws->bo_fence_lock);
121 if (amdgpu_fence_wait(fence, abs_timeout, true))
122 fence_idle = true;
123 else
124 buffer_idle = false;
125 simple_mtx_lock(&ws->bo_fence_lock);
126
127 /* Release an idle fence to avoid checking it again later, keeping in
128 * mind that the fence array may have been modified by other threads.
129 */
130 if (fence_idle && bo->num_fences && bo->fences[0] == fence) {
131 amdgpu_fence_reference(&bo->fences[0], NULL);
132 memmove(&bo->fences[0], &bo->fences[1],
133 (bo->num_fences - 1) * sizeof(*bo->fences));
134 bo->num_fences--;
135 }
136
137 amdgpu_fence_reference(&fence, NULL);
138 }
139 simple_mtx_unlock(&ws->bo_fence_lock);
140
141 return buffer_idle;
142 }
143 }
144
145 static enum radeon_bo_domain amdgpu_bo_get_initial_domain(
146 struct pb_buffer *buf)
147 {
148 return ((struct amdgpu_winsys_bo*)buf)->initial_domain;
149 }
150
151 static enum radeon_bo_flag amdgpu_bo_get_flags(
152 struct pb_buffer *buf)
153 {
154 return ((struct amdgpu_winsys_bo*)buf)->flags;
155 }
156
157 static void amdgpu_bo_remove_fences(struct amdgpu_winsys_bo *bo)
158 {
159 for (unsigned i = 0; i < bo->num_fences; ++i)
160 amdgpu_fence_reference(&bo->fences[i], NULL);
161
162 FREE(bo->fences);
163 bo->num_fences = 0;
164 bo->max_fences = 0;
165 }
166
167 void amdgpu_bo_destroy(struct pb_buffer *_buf)
168 {
169 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
170 struct amdgpu_screen_winsys *sws_iter;
171 struct amdgpu_winsys *ws = bo->ws;
172
173 assert(bo->bo && "must not be called for slab entries");
174
175 if (!bo->is_user_ptr && bo->cpu_ptr) {
176 bo->cpu_ptr = NULL;
177 amdgpu_bo_unmap(&bo->base);
178 }
179 assert(bo->is_user_ptr || bo->u.real.map_count == 0);
180
181 if (ws->debug_all_bos) {
182 simple_mtx_lock(&ws->global_bo_list_lock);
183 list_del(&bo->u.real.global_list_item);
184 ws->num_buffers--;
185 simple_mtx_unlock(&ws->global_bo_list_lock);
186 }
187
188 /* Close all KMS handles retrieved for other DRM file descriptions */
189 simple_mtx_lock(&ws->sws_list_lock);
190 for (sws_iter = ws->sws_list; sws_iter; sws_iter = sws_iter->next) {
191 struct hash_entry *entry;
192
193 if (!sws_iter->kms_handles)
194 continue;
195
196 entry = _mesa_hash_table_search(sws_iter->kms_handles, bo);
197 if (entry) {
198 struct drm_gem_close args = { .handle = (uintptr_t)entry->data };
199
200 drmIoctl(sws_iter->fd, DRM_IOCTL_GEM_CLOSE, &args);
201 _mesa_hash_table_remove(sws_iter->kms_handles, entry);
202 }
203 }
204 simple_mtx_unlock(&ws->sws_list_lock);
205
206 simple_mtx_lock(&ws->bo_export_table_lock);
207 _mesa_hash_table_remove_key(ws->bo_export_table, bo->bo);
208 simple_mtx_unlock(&ws->bo_export_table_lock);
209
210 if (bo->initial_domain & RADEON_DOMAIN_VRAM_GTT) {
211 amdgpu_bo_va_op(bo->bo, 0, bo->base.size, bo->va, 0, AMDGPU_VA_OP_UNMAP);
212 amdgpu_va_range_free(bo->u.real.va_handle);
213 }
214 amdgpu_bo_free(bo->bo);
215
216 amdgpu_bo_remove_fences(bo);
217
218 if (bo->initial_domain & RADEON_DOMAIN_VRAM)
219 ws->allocated_vram -= align64(bo->base.size, ws->info.gart_page_size);
220 else if (bo->initial_domain & RADEON_DOMAIN_GTT)
221 ws->allocated_gtt -= align64(bo->base.size, ws->info.gart_page_size);
222
223 simple_mtx_destroy(&bo->lock);
224 FREE(bo);
225 }
226
227 static void amdgpu_bo_destroy_or_cache(struct pb_buffer *_buf)
228 {
229 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
230
231 assert(bo->bo); /* slab buffers have a separate vtbl */
232
233 if (bo->u.real.use_reusable_pool)
234 pb_cache_add_buffer(&bo->u.real.cache_entry);
235 else
236 amdgpu_bo_destroy(_buf);
237 }
238
239 static void amdgpu_clean_up_buffer_managers(struct amdgpu_winsys *ws)
240 {
241 for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
242 pb_slabs_reclaim(&ws->bo_slabs[i]);
243 if (ws->secure)
244 pb_slabs_reclaim(&ws->bo_slabs_encrypted[i]);
245 }
246
247 pb_cache_release_all_buffers(&ws->bo_cache);
248 }
249
250 static bool amdgpu_bo_do_map(struct amdgpu_winsys_bo *bo, void **cpu)
251 {
252 assert(!bo->sparse && bo->bo && !bo->is_user_ptr);
253 int r = amdgpu_bo_cpu_map(bo->bo, cpu);
254 if (r) {
255 /* Clean up buffer managers and try again. */
256 amdgpu_clean_up_buffer_managers(bo->ws);
257 r = amdgpu_bo_cpu_map(bo->bo, cpu);
258 if (r)
259 return false;
260 }
261
262 if (p_atomic_inc_return(&bo->u.real.map_count) == 1) {
263 if (bo->initial_domain & RADEON_DOMAIN_VRAM)
264 bo->ws->mapped_vram += bo->base.size;
265 else if (bo->initial_domain & RADEON_DOMAIN_GTT)
266 bo->ws->mapped_gtt += bo->base.size;
267 bo->ws->num_mapped_buffers++;
268 }
269
270 return true;
271 }
272
273 void *amdgpu_bo_map(struct pb_buffer *buf,
274 struct radeon_cmdbuf *rcs,
275 enum pipe_transfer_usage usage)
276 {
277 struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
278 struct amdgpu_winsys_bo *real;
279 struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
280
281 assert(!bo->sparse);
282
283 /* If it's not unsynchronized bo_map, flush CS if needed and then wait. */
284 if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
285 /* DONTBLOCK doesn't make sense with UNSYNCHRONIZED. */
286 if (usage & PIPE_TRANSFER_DONTBLOCK) {
287 if (!(usage & PIPE_TRANSFER_WRITE)) {
288 /* Mapping for read.
289 *
290 * Since we are mapping for read, we don't need to wait
291 * if the GPU is using the buffer for read too
292 * (neither one is changing it).
293 *
294 * Only check whether the buffer is being used for write. */
295 if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
296 RADEON_USAGE_WRITE)) {
297 cs->flush_cs(cs->flush_data,
298 RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
299 return NULL;
300 }
301
302 if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
303 RADEON_USAGE_WRITE)) {
304 return NULL;
305 }
306 } else {
307 if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo)) {
308 cs->flush_cs(cs->flush_data,
309 RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
310 return NULL;
311 }
312
313 if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
314 RADEON_USAGE_READWRITE)) {
315 return NULL;
316 }
317 }
318 } else {
319 uint64_t time = os_time_get_nano();
320
321 if (!(usage & PIPE_TRANSFER_WRITE)) {
322 /* Mapping for read.
323 *
324 * Since we are mapping for read, we don't need to wait
325 * if the GPU is using the buffer for read too
326 * (neither one is changing it).
327 *
328 * Only check whether the buffer is being used for write. */
329 if (cs) {
330 if (amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
331 RADEON_USAGE_WRITE)) {
332 cs->flush_cs(cs->flush_data,
333 RADEON_FLUSH_START_NEXT_GFX_IB_NOW, NULL);
334 } else {
335 /* Try to avoid busy-waiting in amdgpu_bo_wait. */
336 if (p_atomic_read(&bo->num_active_ioctls))
337 amdgpu_cs_sync_flush(rcs);
338 }
339 }
340
341 amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
342 RADEON_USAGE_WRITE);
343 } else {
344 /* Mapping for write. */
345 if (cs) {
346 if (amdgpu_bo_is_referenced_by_cs(cs, bo)) {
347 cs->flush_cs(cs->flush_data,
348 RADEON_FLUSH_START_NEXT_GFX_IB_NOW, NULL);
349 } else {
350 /* Try to avoid busy-waiting in amdgpu_bo_wait. */
351 if (p_atomic_read(&bo->num_active_ioctls))
352 amdgpu_cs_sync_flush(rcs);
353 }
354 }
355
356 amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
357 RADEON_USAGE_READWRITE);
358 }
359
360 bo->ws->buffer_wait_time += os_time_get_nano() - time;
361 }
362 }
363
364 /* Buffer synchronization has been checked, now actually map the buffer. */
365 void *cpu = NULL;
366 uint64_t offset = 0;
367
368 if (bo->bo) {
369 real = bo;
370 } else {
371 real = bo->u.slab.real;
372 offset = bo->va - real->va;
373 }
374
375 if (usage & RADEON_TRANSFER_TEMPORARY) {
376 if (real->is_user_ptr) {
377 cpu = real->cpu_ptr;
378 } else {
379 if (!amdgpu_bo_do_map(real, &cpu))
380 return NULL;
381 }
382 } else {
383 cpu = p_atomic_read(&real->cpu_ptr);
384 if (!cpu) {
385 simple_mtx_lock(&real->lock);
386 /* Must re-check due to the possibility of a race. Re-check need not
387 * be atomic thanks to the lock. */
388 cpu = real->cpu_ptr;
389 if (!cpu) {
390 if (!amdgpu_bo_do_map(real, &cpu)) {
391 simple_mtx_unlock(&real->lock);
392 return NULL;
393 }
394 p_atomic_set(&real->cpu_ptr, cpu);
395 }
396 simple_mtx_unlock(&real->lock);
397 }
398 }
399
400 return (uint8_t*)cpu + offset;
401 }
402
403 static void amdgpu_bo_unmap(struct pb_buffer *buf)
404 {
405 struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
406 struct amdgpu_winsys_bo *real;
407
408 assert(!bo->sparse);
409
410 if (bo->is_user_ptr)
411 return;
412
413 real = bo->bo ? bo : bo->u.slab.real;
414 assert(real->u.real.map_count != 0 && "too many unmaps");
415 if (p_atomic_dec_zero(&real->u.real.map_count)) {
416 assert(!real->cpu_ptr &&
417 "too many unmaps or forgot RADEON_TRANSFER_TEMPORARY flag");
418
419 if (real->initial_domain & RADEON_DOMAIN_VRAM)
420 real->ws->mapped_vram -= real->base.size;
421 else if (real->initial_domain & RADEON_DOMAIN_GTT)
422 real->ws->mapped_gtt -= real->base.size;
423 real->ws->num_mapped_buffers--;
424 }
425
426 amdgpu_bo_cpu_unmap(real->bo);
427 }
428
429 static const struct pb_vtbl amdgpu_winsys_bo_vtbl = {
430 amdgpu_bo_destroy_or_cache
431 /* other functions are never called */
432 };
433
434 static void amdgpu_add_buffer_to_global_list(struct amdgpu_winsys_bo *bo)
435 {
436 struct amdgpu_winsys *ws = bo->ws;
437
438 assert(bo->bo);
439
440 if (ws->debug_all_bos) {
441 simple_mtx_lock(&ws->global_bo_list_lock);
442 list_addtail(&bo->u.real.global_list_item, &ws->global_bo_list);
443 ws->num_buffers++;
444 simple_mtx_unlock(&ws->global_bo_list_lock);
445 }
446 }
447
448 static uint64_t amdgpu_get_optimal_vm_alignment(struct amdgpu_winsys *ws,
449 uint64_t size, unsigned alignment)
450 {
451 uint64_t vm_alignment = alignment;
452
453 /* Increase the VM alignment for faster address translation. */
454 if (size >= ws->info.pte_fragment_size)
455 vm_alignment = MAX2(vm_alignment, ws->info.pte_fragment_size);
456
457 /* Gfx9: Increase the VM alignment to the most significant bit set
458 * in the size for faster address translation.
459 */
460 if (ws->info.chip_class >= GFX9) {
461 unsigned msb = util_last_bit64(size); /* 0 = no bit is set */
462 uint64_t msb_alignment = msb ? 1ull << (msb - 1) : 0;
463
464 vm_alignment = MAX2(vm_alignment, msb_alignment);
465 }
466 return vm_alignment;
467 }
468
469 static struct amdgpu_winsys_bo *amdgpu_create_bo(struct amdgpu_winsys *ws,
470 uint64_t size,
471 unsigned alignment,
472 enum radeon_bo_domain initial_domain,
473 unsigned flags,
474 int heap)
475 {
476 struct amdgpu_bo_alloc_request request = {0};
477 amdgpu_bo_handle buf_handle;
478 uint64_t va = 0;
479 struct amdgpu_winsys_bo *bo;
480 amdgpu_va_handle va_handle = NULL;
481 int r;
482
483 /* VRAM or GTT must be specified, but not both at the same time. */
484 assert(util_bitcount(initial_domain & (RADEON_DOMAIN_VRAM_GTT |
485 RADEON_DOMAIN_GDS |
486 RADEON_DOMAIN_OA)) == 1);
487
488 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
489 if (!bo) {
490 return NULL;
491 }
492
493 if (heap >= 0) {
494 pb_cache_init_entry(&ws->bo_cache, &bo->u.real.cache_entry, &bo->base,
495 heap);
496 }
497 request.alloc_size = size;
498 request.phys_alignment = alignment;
499
500 if (initial_domain & RADEON_DOMAIN_VRAM) {
501 request.preferred_heap |= AMDGPU_GEM_DOMAIN_VRAM;
502
503 /* Since VRAM and GTT have almost the same performance on APUs, we could
504 * just set GTT. However, in order to decrease GTT(RAM) usage, which is
505 * shared with the OS, allow VRAM placements too. The idea is not to use
506 * VRAM usefully, but to use it so that it's not unused and wasted.
507 */
508 if (!ws->info.has_dedicated_vram)
509 request.preferred_heap |= AMDGPU_GEM_DOMAIN_GTT;
510 }
511
512 if (initial_domain & RADEON_DOMAIN_GTT)
513 request.preferred_heap |= AMDGPU_GEM_DOMAIN_GTT;
514 if (initial_domain & RADEON_DOMAIN_GDS)
515 request.preferred_heap |= AMDGPU_GEM_DOMAIN_GDS;
516 if (initial_domain & RADEON_DOMAIN_OA)
517 request.preferred_heap |= AMDGPU_GEM_DOMAIN_OA;
518
519 if (flags & RADEON_FLAG_NO_CPU_ACCESS)
520 request.flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
521 if (flags & RADEON_FLAG_GTT_WC)
522 request.flags |= AMDGPU_GEM_CREATE_CPU_GTT_USWC;
523 if (ws->zero_all_vram_allocs &&
524 (request.preferred_heap & AMDGPU_GEM_DOMAIN_VRAM))
525 request.flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
526 if ((flags & RADEON_FLAG_ENCRYPTED) && ws->secure)
527 request.flags |= AMDGPU_GEM_CREATE_ENCRYPTED;
528
529 r = amdgpu_bo_alloc(ws->dev, &request, &buf_handle);
530 if (r) {
531 fprintf(stderr, "amdgpu: Failed to allocate a buffer:\n");
532 fprintf(stderr, "amdgpu: size : %"PRIu64" bytes\n", size);
533 fprintf(stderr, "amdgpu: alignment : %u bytes\n", alignment);
534 fprintf(stderr, "amdgpu: domains : %u\n", initial_domain);
535 fprintf(stderr, "amdgpu: flags : %" PRIx64 "\n", request.flags);
536 goto error_bo_alloc;
537 }
538
539 if (initial_domain & RADEON_DOMAIN_VRAM_GTT) {
540 unsigned va_gap_size = ws->check_vm ? MAX2(4 * alignment, 64 * 1024) : 0;
541
542 r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
543 size + va_gap_size,
544 amdgpu_get_optimal_vm_alignment(ws, size, alignment),
545 0, &va, &va_handle,
546 (flags & RADEON_FLAG_32BIT ? AMDGPU_VA_RANGE_32_BIT : 0) |
547 AMDGPU_VA_RANGE_HIGH);
548 if (r)
549 goto error_va_alloc;
550
551 unsigned vm_flags = AMDGPU_VM_PAGE_READABLE |
552 AMDGPU_VM_PAGE_EXECUTABLE;
553
554 if (!(flags & RADEON_FLAG_READ_ONLY))
555 vm_flags |= AMDGPU_VM_PAGE_WRITEABLE;
556
557 if (flags & RADEON_FLAG_UNCACHED)
558 vm_flags |= AMDGPU_VM_MTYPE_UC;
559
560 r = amdgpu_bo_va_op_raw(ws->dev, buf_handle, 0, size, va, vm_flags,
561 AMDGPU_VA_OP_MAP);
562 if (r)
563 goto error_va_map;
564 }
565
566 simple_mtx_init(&bo->lock, mtx_plain);
567 pipe_reference_init(&bo->base.reference, 1);
568 bo->base.alignment = alignment;
569 bo->base.usage = 0;
570 bo->base.size = size;
571 bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
572 bo->ws = ws;
573 bo->bo = buf_handle;
574 bo->va = va;
575 bo->u.real.va_handle = va_handle;
576 bo->initial_domain = initial_domain;
577 bo->flags = flags;
578 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
579
580 if (initial_domain & RADEON_DOMAIN_VRAM)
581 ws->allocated_vram += align64(size, ws->info.gart_page_size);
582 else if (initial_domain & RADEON_DOMAIN_GTT)
583 ws->allocated_gtt += align64(size, ws->info.gart_page_size);
584
585 amdgpu_bo_export(bo->bo, amdgpu_bo_handle_type_kms, &bo->u.real.kms_handle);
586
587 amdgpu_add_buffer_to_global_list(bo);
588
589 return bo;
590
591 error_va_map:
592 amdgpu_va_range_free(va_handle);
593
594 error_va_alloc:
595 amdgpu_bo_free(buf_handle);
596
597 error_bo_alloc:
598 FREE(bo);
599 return NULL;
600 }
601
602 bool amdgpu_bo_can_reclaim(struct pb_buffer *_buf)
603 {
604 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
605
606 if (amdgpu_bo_is_referenced_by_any_cs(bo)) {
607 return false;
608 }
609
610 return amdgpu_bo_wait(_buf, 0, RADEON_USAGE_READWRITE);
611 }
612
613 bool amdgpu_bo_can_reclaim_slab(void *priv, struct pb_slab_entry *entry)
614 {
615 struct amdgpu_winsys_bo *bo = NULL; /* fix container_of */
616 bo = container_of(entry, bo, u.slab.entry);
617
618 return amdgpu_bo_can_reclaim(&bo->base);
619 }
620
621 static struct pb_slabs *get_slabs(struct amdgpu_winsys *ws, uint64_t size,
622 enum radeon_bo_flag flags)
623 {
624 struct pb_slabs *bo_slabs = ((flags & RADEON_FLAG_ENCRYPTED) && ws->secure) ?
625 ws->bo_slabs_encrypted : ws->bo_slabs;
626 /* Find the correct slab allocator for the given size. */
627 for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
628 struct pb_slabs *slabs = &bo_slabs[i];
629
630 if (size <= 1 << (slabs->min_order + slabs->num_orders - 1))
631 return slabs;
632 }
633
634 assert(0);
635 return NULL;
636 }
637
638 static void amdgpu_bo_slab_destroy(struct pb_buffer *_buf)
639 {
640 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
641
642 assert(!bo->bo);
643
644 if (bo->flags & RADEON_FLAG_ENCRYPTED)
645 pb_slab_free(get_slabs(bo->ws,
646 bo->base.size,
647 RADEON_FLAG_ENCRYPTED), &bo->u.slab.entry);
648 else
649 pb_slab_free(get_slabs(bo->ws,
650 bo->base.size,
651 0), &bo->u.slab.entry);
652 }
653
654 static const struct pb_vtbl amdgpu_winsys_bo_slab_vtbl = {
655 amdgpu_bo_slab_destroy
656 /* other functions are never called */
657 };
658
659 static struct pb_slab *amdgpu_bo_slab_alloc(void *priv, unsigned heap,
660 unsigned entry_size,
661 unsigned group_index,
662 bool encrypted)
663 {
664 struct amdgpu_winsys *ws = priv;
665 struct amdgpu_slab *slab = CALLOC_STRUCT(amdgpu_slab);
666 enum radeon_bo_domain domains = radeon_domain_from_heap(heap);
667 enum radeon_bo_flag flags = radeon_flags_from_heap(heap);
668 uint32_t base_id;
669 unsigned slab_size = 0;
670
671 if (!slab)
672 return NULL;
673
674 if (encrypted)
675 flags |= RADEON_FLAG_ENCRYPTED;
676
677 struct pb_slabs *slabs = (flags & RADEON_FLAG_ENCRYPTED && ws->secure) ?
678 ws->bo_slabs_encrypted : ws->bo_slabs;
679
680 /* Determine the slab buffer size. */
681 for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
682 unsigned max_entry_size = 1 << (slabs[i].min_order + slabs[i].num_orders - 1);
683
684 if (entry_size <= max_entry_size) {
685 /* The slab size is twice the size of the largest possible entry. */
686 slab_size = max_entry_size * 2;
687
688 /* The largest slab should have the same size as the PTE fragment
689 * size to get faster address translation.
690 */
691 if (i == NUM_SLAB_ALLOCATORS - 1 &&
692 slab_size < ws->info.pte_fragment_size)
693 slab_size = ws->info.pte_fragment_size;
694 break;
695 }
696 }
697 assert(slab_size != 0);
698
699 slab->buffer = amdgpu_winsys_bo(amdgpu_bo_create(ws,
700 slab_size, slab_size,
701 domains, flags));
702 if (!slab->buffer)
703 goto fail;
704
705 slab->base.num_entries = slab->buffer->base.size / entry_size;
706 slab->base.num_free = slab->base.num_entries;
707 slab->entries = CALLOC(slab->base.num_entries, sizeof(*slab->entries));
708 if (!slab->entries)
709 goto fail_buffer;
710
711 list_inithead(&slab->base.free);
712
713 base_id = __sync_fetch_and_add(&ws->next_bo_unique_id, slab->base.num_entries);
714
715 for (unsigned i = 0; i < slab->base.num_entries; ++i) {
716 struct amdgpu_winsys_bo *bo = &slab->entries[i];
717
718 simple_mtx_init(&bo->lock, mtx_plain);
719 bo->base.alignment = entry_size;
720 bo->base.usage = slab->buffer->base.usage;
721 bo->base.size = entry_size;
722 bo->base.vtbl = &amdgpu_winsys_bo_slab_vtbl;
723 bo->ws = ws;
724 bo->va = slab->buffer->va + i * entry_size;
725 bo->initial_domain = domains;
726 bo->unique_id = base_id + i;
727 bo->u.slab.entry.slab = &slab->base;
728 bo->u.slab.entry.group_index = group_index;
729
730 if (slab->buffer->bo) {
731 /* The slab is not suballocated. */
732 bo->u.slab.real = slab->buffer;
733 } else {
734 /* The slab is allocated out of a bigger slab. */
735 bo->u.slab.real = slab->buffer->u.slab.real;
736 assert(bo->u.slab.real->bo);
737 }
738
739 list_addtail(&bo->u.slab.entry.head, &slab->base.free);
740 }
741
742 return &slab->base;
743
744 fail_buffer:
745 amdgpu_winsys_bo_reference(&slab->buffer, NULL);
746 fail:
747 FREE(slab);
748 return NULL;
749 }
750
751 struct pb_slab *amdgpu_bo_slab_alloc_encrypted(void *priv, unsigned heap,
752 unsigned entry_size,
753 unsigned group_index)
754 {
755 return amdgpu_bo_slab_alloc(priv, heap, entry_size, group_index, true);
756 }
757
758 struct pb_slab *amdgpu_bo_slab_alloc_normal(void *priv, unsigned heap,
759 unsigned entry_size,
760 unsigned group_index)
761 {
762 return amdgpu_bo_slab_alloc(priv, heap, entry_size, group_index, false);
763 }
764
765 void amdgpu_bo_slab_free(void *priv, struct pb_slab *pslab)
766 {
767 struct amdgpu_slab *slab = amdgpu_slab(pslab);
768
769 for (unsigned i = 0; i < slab->base.num_entries; ++i) {
770 amdgpu_bo_remove_fences(&slab->entries[i]);
771 simple_mtx_destroy(&slab->entries[i].lock);
772 }
773
774 FREE(slab->entries);
775 amdgpu_winsys_bo_reference(&slab->buffer, NULL);
776 FREE(slab);
777 }
778
779 #if DEBUG_SPARSE_COMMITS
780 static void
781 sparse_dump(struct amdgpu_winsys_bo *bo, const char *func)
782 {
783 fprintf(stderr, "%s: %p (size=%"PRIu64", num_va_pages=%u) @ %s\n"
784 "Commitments:\n",
785 __func__, bo, bo->base.size, bo->u.sparse.num_va_pages, func);
786
787 struct amdgpu_sparse_backing *span_backing = NULL;
788 uint32_t span_first_backing_page = 0;
789 uint32_t span_first_va_page = 0;
790 uint32_t va_page = 0;
791
792 for (;;) {
793 struct amdgpu_sparse_backing *backing = 0;
794 uint32_t backing_page = 0;
795
796 if (va_page < bo->u.sparse.num_va_pages) {
797 backing = bo->u.sparse.commitments[va_page].backing;
798 backing_page = bo->u.sparse.commitments[va_page].page;
799 }
800
801 if (span_backing &&
802 (backing != span_backing ||
803 backing_page != span_first_backing_page + (va_page - span_first_va_page))) {
804 fprintf(stderr, " %u..%u: backing=%p:%u..%u\n",
805 span_first_va_page, va_page - 1, span_backing,
806 span_first_backing_page,
807 span_first_backing_page + (va_page - span_first_va_page) - 1);
808
809 span_backing = NULL;
810 }
811
812 if (va_page >= bo->u.sparse.num_va_pages)
813 break;
814
815 if (backing && !span_backing) {
816 span_backing = backing;
817 span_first_backing_page = backing_page;
818 span_first_va_page = va_page;
819 }
820
821 va_page++;
822 }
823
824 fprintf(stderr, "Backing:\n");
825
826 list_for_each_entry(struct amdgpu_sparse_backing, backing, &bo->u.sparse.backing, list) {
827 fprintf(stderr, " %p (size=%"PRIu64")\n", backing, backing->bo->base.size);
828 for (unsigned i = 0; i < backing->num_chunks; ++i)
829 fprintf(stderr, " %u..%u\n", backing->chunks[i].begin, backing->chunks[i].end);
830 }
831 }
832 #endif
833
834 /*
835 * Attempt to allocate the given number of backing pages. Fewer pages may be
836 * allocated (depending on the fragmentation of existing backing buffers),
837 * which will be reflected by a change to *pnum_pages.
838 */
839 static struct amdgpu_sparse_backing *
840 sparse_backing_alloc(struct amdgpu_winsys_bo *bo, uint32_t *pstart_page, uint32_t *pnum_pages)
841 {
842 struct amdgpu_sparse_backing *best_backing;
843 unsigned best_idx;
844 uint32_t best_num_pages;
845
846 best_backing = NULL;
847 best_idx = 0;
848 best_num_pages = 0;
849
850 /* This is a very simple and inefficient best-fit algorithm. */
851 list_for_each_entry(struct amdgpu_sparse_backing, backing, &bo->u.sparse.backing, list) {
852 for (unsigned idx = 0; idx < backing->num_chunks; ++idx) {
853 uint32_t cur_num_pages = backing->chunks[idx].end - backing->chunks[idx].begin;
854 if ((best_num_pages < *pnum_pages && cur_num_pages > best_num_pages) ||
855 (best_num_pages > *pnum_pages && cur_num_pages < best_num_pages)) {
856 best_backing = backing;
857 best_idx = idx;
858 best_num_pages = cur_num_pages;
859 }
860 }
861 }
862
863 /* Allocate a new backing buffer if necessary. */
864 if (!best_backing) {
865 struct pb_buffer *buf;
866 uint64_t size;
867 uint32_t pages;
868
869 best_backing = CALLOC_STRUCT(amdgpu_sparse_backing);
870 if (!best_backing)
871 return NULL;
872
873 best_backing->max_chunks = 4;
874 best_backing->chunks = CALLOC(best_backing->max_chunks,
875 sizeof(*best_backing->chunks));
876 if (!best_backing->chunks) {
877 FREE(best_backing);
878 return NULL;
879 }
880
881 assert(bo->u.sparse.num_backing_pages < DIV_ROUND_UP(bo->base.size, RADEON_SPARSE_PAGE_SIZE));
882
883 size = MIN3(bo->base.size / 16,
884 8 * 1024 * 1024,
885 bo->base.size - (uint64_t)bo->u.sparse.num_backing_pages * RADEON_SPARSE_PAGE_SIZE);
886 size = MAX2(size, RADEON_SPARSE_PAGE_SIZE);
887
888 buf = amdgpu_bo_create(bo->ws, size, RADEON_SPARSE_PAGE_SIZE,
889 bo->initial_domain,
890 bo->u.sparse.flags | RADEON_FLAG_NO_SUBALLOC);
891 if (!buf) {
892 FREE(best_backing->chunks);
893 FREE(best_backing);
894 return NULL;
895 }
896
897 /* We might have gotten a bigger buffer than requested via caching. */
898 pages = buf->size / RADEON_SPARSE_PAGE_SIZE;
899
900 best_backing->bo = amdgpu_winsys_bo(buf);
901 best_backing->num_chunks = 1;
902 best_backing->chunks[0].begin = 0;
903 best_backing->chunks[0].end = pages;
904
905 list_add(&best_backing->list, &bo->u.sparse.backing);
906 bo->u.sparse.num_backing_pages += pages;
907
908 best_idx = 0;
909 best_num_pages = pages;
910 }
911
912 *pnum_pages = MIN2(*pnum_pages, best_num_pages);
913 *pstart_page = best_backing->chunks[best_idx].begin;
914 best_backing->chunks[best_idx].begin += *pnum_pages;
915
916 if (best_backing->chunks[best_idx].begin >= best_backing->chunks[best_idx].end) {
917 memmove(&best_backing->chunks[best_idx], &best_backing->chunks[best_idx + 1],
918 sizeof(*best_backing->chunks) * (best_backing->num_chunks - best_idx - 1));
919 best_backing->num_chunks--;
920 }
921
922 return best_backing;
923 }
924
925 static void
926 sparse_free_backing_buffer(struct amdgpu_winsys_bo *bo,
927 struct amdgpu_sparse_backing *backing)
928 {
929 struct amdgpu_winsys *ws = backing->bo->ws;
930
931 bo->u.sparse.num_backing_pages -= backing->bo->base.size / RADEON_SPARSE_PAGE_SIZE;
932
933 simple_mtx_lock(&ws->bo_fence_lock);
934 amdgpu_add_fences(backing->bo, bo->num_fences, bo->fences);
935 simple_mtx_unlock(&ws->bo_fence_lock);
936
937 list_del(&backing->list);
938 amdgpu_winsys_bo_reference(&backing->bo, NULL);
939 FREE(backing->chunks);
940 FREE(backing);
941 }
942
943 /*
944 * Return a range of pages from the given backing buffer back into the
945 * free structure.
946 */
947 static bool
948 sparse_backing_free(struct amdgpu_winsys_bo *bo,
949 struct amdgpu_sparse_backing *backing,
950 uint32_t start_page, uint32_t num_pages)
951 {
952 uint32_t end_page = start_page + num_pages;
953 unsigned low = 0;
954 unsigned high = backing->num_chunks;
955
956 /* Find the first chunk with begin >= start_page. */
957 while (low < high) {
958 unsigned mid = low + (high - low) / 2;
959
960 if (backing->chunks[mid].begin >= start_page)
961 high = mid;
962 else
963 low = mid + 1;
964 }
965
966 assert(low >= backing->num_chunks || end_page <= backing->chunks[low].begin);
967 assert(low == 0 || backing->chunks[low - 1].end <= start_page);
968
969 if (low > 0 && backing->chunks[low - 1].end == start_page) {
970 backing->chunks[low - 1].end = end_page;
971
972 if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
973 backing->chunks[low - 1].end = backing->chunks[low].end;
974 memmove(&backing->chunks[low], &backing->chunks[low + 1],
975 sizeof(*backing->chunks) * (backing->num_chunks - low - 1));
976 backing->num_chunks--;
977 }
978 } else if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
979 backing->chunks[low].begin = start_page;
980 } else {
981 if (backing->num_chunks >= backing->max_chunks) {
982 unsigned new_max_chunks = 2 * backing->max_chunks;
983 struct amdgpu_sparse_backing_chunk *new_chunks =
984 REALLOC(backing->chunks,
985 sizeof(*backing->chunks) * backing->max_chunks,
986 sizeof(*backing->chunks) * new_max_chunks);
987 if (!new_chunks)
988 return false;
989
990 backing->max_chunks = new_max_chunks;
991 backing->chunks = new_chunks;
992 }
993
994 memmove(&backing->chunks[low + 1], &backing->chunks[low],
995 sizeof(*backing->chunks) * (backing->num_chunks - low));
996 backing->chunks[low].begin = start_page;
997 backing->chunks[low].end = end_page;
998 backing->num_chunks++;
999 }
1000
1001 if (backing->num_chunks == 1 && backing->chunks[0].begin == 0 &&
1002 backing->chunks[0].end == backing->bo->base.size / RADEON_SPARSE_PAGE_SIZE)
1003 sparse_free_backing_buffer(bo, backing);
1004
1005 return true;
1006 }
1007
1008 static void amdgpu_bo_sparse_destroy(struct pb_buffer *_buf)
1009 {
1010 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
1011 int r;
1012
1013 assert(!bo->bo && bo->sparse);
1014
1015 r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0,
1016 (uint64_t)bo->u.sparse.num_va_pages * RADEON_SPARSE_PAGE_SIZE,
1017 bo->va, 0, AMDGPU_VA_OP_CLEAR);
1018 if (r) {
1019 fprintf(stderr, "amdgpu: clearing PRT VA region on destroy failed (%d)\n", r);
1020 }
1021
1022 while (!list_is_empty(&bo->u.sparse.backing)) {
1023 struct amdgpu_sparse_backing *dummy = NULL;
1024 sparse_free_backing_buffer(bo,
1025 container_of(bo->u.sparse.backing.next,
1026 dummy, list));
1027 }
1028
1029 amdgpu_va_range_free(bo->u.sparse.va_handle);
1030 FREE(bo->u.sparse.commitments);
1031 simple_mtx_destroy(&bo->lock);
1032 FREE(bo);
1033 }
1034
1035 static const struct pb_vtbl amdgpu_winsys_bo_sparse_vtbl = {
1036 amdgpu_bo_sparse_destroy
1037 /* other functions are never called */
1038 };
1039
1040 static struct pb_buffer *
1041 amdgpu_bo_sparse_create(struct amdgpu_winsys *ws, uint64_t size,
1042 enum radeon_bo_domain domain,
1043 enum radeon_bo_flag flags)
1044 {
1045 struct amdgpu_winsys_bo *bo;
1046 uint64_t map_size;
1047 uint64_t va_gap_size;
1048 int r;
1049
1050 /* We use 32-bit page numbers; refuse to attempt allocating sparse buffers
1051 * that exceed this limit. This is not really a restriction: we don't have
1052 * that much virtual address space anyway.
1053 */
1054 if (size > (uint64_t)INT32_MAX * RADEON_SPARSE_PAGE_SIZE)
1055 return NULL;
1056
1057 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
1058 if (!bo)
1059 return NULL;
1060
1061 simple_mtx_init(&bo->lock, mtx_plain);
1062 pipe_reference_init(&bo->base.reference, 1);
1063 bo->base.alignment = RADEON_SPARSE_PAGE_SIZE;
1064 bo->base.size = size;
1065 bo->base.vtbl = &amdgpu_winsys_bo_sparse_vtbl;
1066 bo->ws = ws;
1067 bo->initial_domain = domain;
1068 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
1069 bo->sparse = true;
1070 bo->u.sparse.flags = flags & ~RADEON_FLAG_SPARSE;
1071
1072 bo->u.sparse.num_va_pages = DIV_ROUND_UP(size, RADEON_SPARSE_PAGE_SIZE);
1073 bo->u.sparse.commitments = CALLOC(bo->u.sparse.num_va_pages,
1074 sizeof(*bo->u.sparse.commitments));
1075 if (!bo->u.sparse.commitments)
1076 goto error_alloc_commitments;
1077
1078 list_inithead(&bo->u.sparse.backing);
1079
1080 /* For simplicity, we always map a multiple of the page size. */
1081 map_size = align64(size, RADEON_SPARSE_PAGE_SIZE);
1082 va_gap_size = ws->check_vm ? 4 * RADEON_SPARSE_PAGE_SIZE : 0;
1083 r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
1084 map_size + va_gap_size, RADEON_SPARSE_PAGE_SIZE,
1085 0, &bo->va, &bo->u.sparse.va_handle,
1086 AMDGPU_VA_RANGE_HIGH);
1087 if (r)
1088 goto error_va_alloc;
1089
1090 r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0, size, bo->va,
1091 AMDGPU_VM_PAGE_PRT, AMDGPU_VA_OP_MAP);
1092 if (r)
1093 goto error_va_map;
1094
1095 return &bo->base;
1096
1097 error_va_map:
1098 amdgpu_va_range_free(bo->u.sparse.va_handle);
1099 error_va_alloc:
1100 FREE(bo->u.sparse.commitments);
1101 error_alloc_commitments:
1102 simple_mtx_destroy(&bo->lock);
1103 FREE(bo);
1104 return NULL;
1105 }
1106
1107 static bool
1108 amdgpu_bo_sparse_commit(struct pb_buffer *buf, uint64_t offset, uint64_t size,
1109 bool commit)
1110 {
1111 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(buf);
1112 struct amdgpu_sparse_commitment *comm;
1113 uint32_t va_page, end_va_page;
1114 bool ok = true;
1115 int r;
1116
1117 assert(bo->sparse);
1118 assert(offset % RADEON_SPARSE_PAGE_SIZE == 0);
1119 assert(offset <= bo->base.size);
1120 assert(size <= bo->base.size - offset);
1121 assert(size % RADEON_SPARSE_PAGE_SIZE == 0 || offset + size == bo->base.size);
1122
1123 comm = bo->u.sparse.commitments;
1124 va_page = offset / RADEON_SPARSE_PAGE_SIZE;
1125 end_va_page = va_page + DIV_ROUND_UP(size, RADEON_SPARSE_PAGE_SIZE);
1126
1127 simple_mtx_lock(&bo->lock);
1128
1129 #if DEBUG_SPARSE_COMMITS
1130 sparse_dump(bo, __func__);
1131 #endif
1132
1133 if (commit) {
1134 while (va_page < end_va_page) {
1135 uint32_t span_va_page;
1136
1137 /* Skip pages that are already committed. */
1138 if (comm[va_page].backing) {
1139 va_page++;
1140 continue;
1141 }
1142
1143 /* Determine length of uncommitted span. */
1144 span_va_page = va_page;
1145 while (va_page < end_va_page && !comm[va_page].backing)
1146 va_page++;
1147
1148 /* Fill the uncommitted span with chunks of backing memory. */
1149 while (span_va_page < va_page) {
1150 struct amdgpu_sparse_backing *backing;
1151 uint32_t backing_start, backing_size;
1152
1153 backing_size = va_page - span_va_page;
1154 backing = sparse_backing_alloc(bo, &backing_start, &backing_size);
1155 if (!backing) {
1156 ok = false;
1157 goto out;
1158 }
1159
1160 r = amdgpu_bo_va_op_raw(bo->ws->dev, backing->bo->bo,
1161 (uint64_t)backing_start * RADEON_SPARSE_PAGE_SIZE,
1162 (uint64_t)backing_size * RADEON_SPARSE_PAGE_SIZE,
1163 bo->va + (uint64_t)span_va_page * RADEON_SPARSE_PAGE_SIZE,
1164 AMDGPU_VM_PAGE_READABLE |
1165 AMDGPU_VM_PAGE_WRITEABLE |
1166 AMDGPU_VM_PAGE_EXECUTABLE,
1167 AMDGPU_VA_OP_REPLACE);
1168 if (r) {
1169 ok = sparse_backing_free(bo, backing, backing_start, backing_size);
1170 assert(ok && "sufficient memory should already be allocated");
1171
1172 ok = false;
1173 goto out;
1174 }
1175
1176 while (backing_size) {
1177 comm[span_va_page].backing = backing;
1178 comm[span_va_page].page = backing_start;
1179 span_va_page++;
1180 backing_start++;
1181 backing_size--;
1182 }
1183 }
1184 }
1185 } else {
1186 r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0,
1187 (uint64_t)(end_va_page - va_page) * RADEON_SPARSE_PAGE_SIZE,
1188 bo->va + (uint64_t)va_page * RADEON_SPARSE_PAGE_SIZE,
1189 AMDGPU_VM_PAGE_PRT, AMDGPU_VA_OP_REPLACE);
1190 if (r) {
1191 ok = false;
1192 goto out;
1193 }
1194
1195 while (va_page < end_va_page) {
1196 struct amdgpu_sparse_backing *backing;
1197 uint32_t backing_start;
1198 uint32_t span_pages;
1199
1200 /* Skip pages that are already uncommitted. */
1201 if (!comm[va_page].backing) {
1202 va_page++;
1203 continue;
1204 }
1205
1206 /* Group contiguous spans of pages. */
1207 backing = comm[va_page].backing;
1208 backing_start = comm[va_page].page;
1209 comm[va_page].backing = NULL;
1210
1211 span_pages = 1;
1212 va_page++;
1213
1214 while (va_page < end_va_page &&
1215 comm[va_page].backing == backing &&
1216 comm[va_page].page == backing_start + span_pages) {
1217 comm[va_page].backing = NULL;
1218 va_page++;
1219 span_pages++;
1220 }
1221
1222 if (!sparse_backing_free(bo, backing, backing_start, span_pages)) {
1223 /* Couldn't allocate tracking data structures, so we have to leak */
1224 fprintf(stderr, "amdgpu: leaking PRT backing memory\n");
1225 ok = false;
1226 }
1227 }
1228 }
1229 out:
1230
1231 simple_mtx_unlock(&bo->lock);
1232
1233 return ok;
1234 }
1235
1236 static void amdgpu_buffer_get_metadata(struct pb_buffer *_buf,
1237 struct radeon_bo_metadata *md,
1238 struct radeon_surf *surf)
1239 {
1240 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
1241 struct amdgpu_bo_info info = {0};
1242 int r;
1243
1244 assert(bo->bo && "must not be called for slab entries");
1245
1246 r = amdgpu_bo_query_info(bo->bo, &info);
1247 if (r)
1248 return;
1249
1250 ac_surface_set_bo_metadata(&bo->ws->info, surf, info.metadata.tiling_info,
1251 &md->mode);
1252
1253 md->size_metadata = info.metadata.size_metadata;
1254 memcpy(md->metadata, info.metadata.umd_metadata, sizeof(md->metadata));
1255 }
1256
1257 static void amdgpu_buffer_set_metadata(struct pb_buffer *_buf,
1258 struct radeon_bo_metadata *md,
1259 struct radeon_surf *surf)
1260 {
1261 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
1262 struct amdgpu_bo_metadata metadata = {0};
1263
1264 assert(bo->bo && "must not be called for slab entries");
1265
1266 ac_surface_get_bo_metadata(&bo->ws->info, surf, &metadata.tiling_info);
1267
1268 metadata.size_metadata = md->size_metadata;
1269 memcpy(metadata.umd_metadata, md->metadata, sizeof(md->metadata));
1270
1271 amdgpu_bo_set_metadata(bo->bo, &metadata);
1272 }
1273
1274 struct pb_buffer *
1275 amdgpu_bo_create(struct amdgpu_winsys *ws,
1276 uint64_t size,
1277 unsigned alignment,
1278 enum radeon_bo_domain domain,
1279 enum radeon_bo_flag flags)
1280 {
1281 struct amdgpu_winsys_bo *bo;
1282 int heap = -1;
1283
1284 if (domain & (RADEON_DOMAIN_GDS | RADEON_DOMAIN_OA))
1285 flags |= RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_NO_SUBALLOC;
1286
1287 /* VRAM implies WC. This is not optional. */
1288 assert(!(domain & RADEON_DOMAIN_VRAM) || flags & RADEON_FLAG_GTT_WC);
1289
1290 /* NO_CPU_ACCESS is not valid with GTT. */
1291 assert(!(domain & RADEON_DOMAIN_GTT) || !(flags & RADEON_FLAG_NO_CPU_ACCESS));
1292
1293 /* Sparse buffers must have NO_CPU_ACCESS set. */
1294 assert(!(flags & RADEON_FLAG_SPARSE) || flags & RADEON_FLAG_NO_CPU_ACCESS);
1295
1296 struct pb_slabs *slabs = (flags & RADEON_FLAG_ENCRYPTED && ws->secure) ?
1297 ws->bo_slabs_encrypted : ws->bo_slabs;
1298 struct pb_slabs *last_slab = &slabs[NUM_SLAB_ALLOCATORS - 1];
1299 unsigned max_slab_entry_size = 1 << (last_slab->min_order + last_slab->num_orders - 1);
1300
1301 /* Sub-allocate small buffers from slabs. */
1302 if (!(flags & (RADEON_FLAG_NO_SUBALLOC | RADEON_FLAG_SPARSE)) &&
1303 size <= max_slab_entry_size &&
1304 /* The alignment must be at most the size of the smallest slab entry or
1305 * the next power of two. */
1306 alignment <= MAX2(1 << slabs[0].min_order, util_next_power_of_two(size))) {
1307 struct pb_slab_entry *entry;
1308 int heap = radeon_get_heap_index(domain, flags);
1309
1310 if (heap < 0 || heap >= RADEON_MAX_SLAB_HEAPS)
1311 goto no_slab;
1312
1313 struct pb_slabs *slabs = get_slabs(ws, size, flags);
1314 entry = pb_slab_alloc(slabs, size, heap);
1315 if (!entry) {
1316 /* Clean up buffer managers and try again. */
1317 amdgpu_clean_up_buffer_managers(ws);
1318
1319 entry = pb_slab_alloc(slabs, size, heap);
1320 }
1321 if (!entry)
1322 return NULL;
1323
1324 bo = NULL;
1325 bo = container_of(entry, bo, u.slab.entry);
1326
1327 pipe_reference_init(&bo->base.reference, 1);
1328
1329 return &bo->base;
1330 }
1331 no_slab:
1332
1333 if (flags & RADEON_FLAG_SPARSE) {
1334 assert(RADEON_SPARSE_PAGE_SIZE % alignment == 0);
1335
1336 return amdgpu_bo_sparse_create(ws, size, domain, flags);
1337 }
1338
1339 /* This flag is irrelevant for the cache. */
1340 flags &= ~RADEON_FLAG_NO_SUBALLOC;
1341
1342 /* Align size to page size. This is the minimum alignment for normal
1343 * BOs. Aligning this here helps the cached bufmgr. Especially small BOs,
1344 * like constant/uniform buffers, can benefit from better and more reuse.
1345 */
1346 if (domain & RADEON_DOMAIN_VRAM_GTT) {
1347 size = align64(size, ws->info.gart_page_size);
1348 alignment = align(alignment, ws->info.gart_page_size);
1349 }
1350
1351 bool use_reusable_pool = flags & RADEON_FLAG_NO_INTERPROCESS_SHARING;
1352
1353 if (use_reusable_pool) {
1354 heap = radeon_get_heap_index(domain, flags & ~RADEON_FLAG_ENCRYPTED);
1355 assert(heap >= 0 && heap < RADEON_MAX_CACHED_HEAPS);
1356
1357 /* Get a buffer from the cache. */
1358 bo = (struct amdgpu_winsys_bo*)
1359 pb_cache_reclaim_buffer(&ws->bo_cache, size, alignment, 0, heap);
1360 if (bo)
1361 return &bo->base;
1362 }
1363
1364 /* Create a new one. */
1365 bo = amdgpu_create_bo(ws, size, alignment, domain, flags, heap);
1366 if (!bo) {
1367 /* Clean up buffer managers and try again. */
1368 amdgpu_clean_up_buffer_managers(ws);
1369
1370 bo = amdgpu_create_bo(ws, size, alignment, domain, flags, heap);
1371 if (!bo)
1372 return NULL;
1373 }
1374
1375 bo->u.real.use_reusable_pool = use_reusable_pool;
1376 return &bo->base;
1377 }
1378
1379 static struct pb_buffer *
1380 amdgpu_buffer_create(struct radeon_winsys *ws,
1381 uint64_t size,
1382 unsigned alignment,
1383 enum radeon_bo_domain domain,
1384 enum radeon_bo_flag flags)
1385 {
1386 struct pb_buffer * res = amdgpu_bo_create(amdgpu_winsys(ws), size, alignment, domain,
1387 flags);
1388 return res;
1389 }
1390
1391 static struct pb_buffer *amdgpu_bo_from_handle(struct radeon_winsys *rws,
1392 struct winsys_handle *whandle,
1393 unsigned vm_alignment)
1394 {
1395 struct amdgpu_winsys *ws = amdgpu_winsys(rws);
1396 struct amdgpu_winsys_bo *bo = NULL;
1397 enum amdgpu_bo_handle_type type;
1398 struct amdgpu_bo_import_result result = {0};
1399 uint64_t va;
1400 amdgpu_va_handle va_handle = NULL;
1401 struct amdgpu_bo_info info = {0};
1402 enum radeon_bo_domain initial = 0;
1403 enum radeon_bo_flag flags = 0;
1404 int r;
1405
1406 switch (whandle->type) {
1407 case WINSYS_HANDLE_TYPE_SHARED:
1408 type = amdgpu_bo_handle_type_gem_flink_name;
1409 break;
1410 case WINSYS_HANDLE_TYPE_FD:
1411 type = amdgpu_bo_handle_type_dma_buf_fd;
1412 break;
1413 default:
1414 return NULL;
1415 }
1416
1417 r = amdgpu_bo_import(ws->dev, type, whandle->handle, &result);
1418 if (r)
1419 return NULL;
1420
1421 simple_mtx_lock(&ws->bo_export_table_lock);
1422 bo = util_hash_table_get(ws->bo_export_table, result.buf_handle);
1423
1424 /* If the amdgpu_winsys_bo instance already exists, bump the reference
1425 * counter and return it.
1426 */
1427 if (bo) {
1428 p_atomic_inc(&bo->base.reference.count);
1429 simple_mtx_unlock(&ws->bo_export_table_lock);
1430
1431 /* Release the buffer handle, because we don't need it anymore.
1432 * This function is returning an existing buffer, which has its own
1433 * handle.
1434 */
1435 amdgpu_bo_free(result.buf_handle);
1436 return &bo->base;
1437 }
1438
1439 /* Get initial domains. */
1440 r = amdgpu_bo_query_info(result.buf_handle, &info);
1441 if (r)
1442 goto error;
1443
1444 r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
1445 result.alloc_size,
1446 amdgpu_get_optimal_vm_alignment(ws, result.alloc_size,
1447 vm_alignment),
1448 0, &va, &va_handle, AMDGPU_VA_RANGE_HIGH);
1449 if (r)
1450 goto error;
1451
1452 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
1453 if (!bo)
1454 goto error;
1455
1456 r = amdgpu_bo_va_op(result.buf_handle, 0, result.alloc_size, va, 0, AMDGPU_VA_OP_MAP);
1457 if (r)
1458 goto error;
1459
1460 if (info.preferred_heap & AMDGPU_GEM_DOMAIN_VRAM)
1461 initial |= RADEON_DOMAIN_VRAM;
1462 if (info.preferred_heap & AMDGPU_GEM_DOMAIN_GTT)
1463 initial |= RADEON_DOMAIN_GTT;
1464 if (info.alloc_flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
1465 flags |= RADEON_FLAG_NO_CPU_ACCESS;
1466 if (info.alloc_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
1467 flags |= RADEON_FLAG_GTT_WC;
1468 if (info.alloc_flags & AMDGPU_GEM_CREATE_ENCRYPTED)
1469 flags |= RADEON_FLAG_ENCRYPTED;
1470
1471 /* Initialize the structure. */
1472 simple_mtx_init(&bo->lock, mtx_plain);
1473 pipe_reference_init(&bo->base.reference, 1);
1474 bo->base.alignment = info.phys_alignment;
1475 bo->bo = result.buf_handle;
1476 bo->base.size = result.alloc_size;
1477 bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
1478 bo->ws = ws;
1479 bo->va = va;
1480 bo->u.real.va_handle = va_handle;
1481 bo->initial_domain = initial;
1482 bo->flags = flags;
1483 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
1484 bo->is_shared = true;
1485
1486 if (bo->initial_domain & RADEON_DOMAIN_VRAM)
1487 ws->allocated_vram += align64(bo->base.size, ws->info.gart_page_size);
1488 else if (bo->initial_domain & RADEON_DOMAIN_GTT)
1489 ws->allocated_gtt += align64(bo->base.size, ws->info.gart_page_size);
1490
1491 amdgpu_bo_export(bo->bo, amdgpu_bo_handle_type_kms, &bo->u.real.kms_handle);
1492
1493 amdgpu_add_buffer_to_global_list(bo);
1494
1495 _mesa_hash_table_insert(ws->bo_export_table, bo->bo, bo);
1496 simple_mtx_unlock(&ws->bo_export_table_lock);
1497
1498 return &bo->base;
1499
1500 error:
1501 simple_mtx_unlock(&ws->bo_export_table_lock);
1502 if (bo)
1503 FREE(bo);
1504 if (va_handle)
1505 amdgpu_va_range_free(va_handle);
1506 amdgpu_bo_free(result.buf_handle);
1507 return NULL;
1508 }
1509
1510 static bool amdgpu_bo_get_handle(struct radeon_winsys *rws,
1511 struct pb_buffer *buffer,
1512 struct winsys_handle *whandle)
1513 {
1514 struct amdgpu_screen_winsys *sws = amdgpu_screen_winsys(rws);
1515 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(buffer);
1516 struct amdgpu_winsys *ws = bo->ws;
1517 enum amdgpu_bo_handle_type type;
1518 struct hash_entry *entry;
1519 int r;
1520
1521 /* Don't allow exports of slab entries and sparse buffers. */
1522 if (!bo->bo)
1523 return false;
1524
1525 bo->u.real.use_reusable_pool = false;
1526
1527 switch (whandle->type) {
1528 case WINSYS_HANDLE_TYPE_SHARED:
1529 type = amdgpu_bo_handle_type_gem_flink_name;
1530 break;
1531 case WINSYS_HANDLE_TYPE_KMS:
1532 if (sws->fd == ws->fd) {
1533 whandle->handle = bo->u.real.kms_handle;
1534
1535 if (bo->is_shared)
1536 return true;
1537
1538 goto hash_table_set;
1539 }
1540
1541 simple_mtx_lock(&ws->sws_list_lock);
1542 entry = _mesa_hash_table_search(sws->kms_handles, bo);
1543 simple_mtx_unlock(&ws->sws_list_lock);
1544 if (entry) {
1545 whandle->handle = (uintptr_t)entry->data;
1546 return true;
1547 }
1548 /* Fall through */
1549 case WINSYS_HANDLE_TYPE_FD:
1550 type = amdgpu_bo_handle_type_dma_buf_fd;
1551 break;
1552 default:
1553 return false;
1554 }
1555
1556 r = amdgpu_bo_export(bo->bo, type, &whandle->handle);
1557 if (r)
1558 return false;
1559
1560 if (whandle->type == WINSYS_HANDLE_TYPE_KMS) {
1561 int dma_fd = whandle->handle;
1562
1563 r = drmPrimeFDToHandle(sws->fd, dma_fd, &whandle->handle);
1564 close(dma_fd);
1565
1566 if (r)
1567 return false;
1568
1569 simple_mtx_lock(&ws->sws_list_lock);
1570 _mesa_hash_table_insert_pre_hashed(sws->kms_handles,
1571 bo->u.real.kms_handle, bo,
1572 (void*)(uintptr_t)whandle->handle);
1573 simple_mtx_unlock(&ws->sws_list_lock);
1574 }
1575
1576 hash_table_set:
1577 simple_mtx_lock(&ws->bo_export_table_lock);
1578 _mesa_hash_table_insert(ws->bo_export_table, bo->bo, bo);
1579 simple_mtx_unlock(&ws->bo_export_table_lock);
1580
1581 bo->is_shared = true;
1582 return true;
1583 }
1584
1585 static struct pb_buffer *amdgpu_bo_from_ptr(struct radeon_winsys *rws,
1586 void *pointer, uint64_t size)
1587 {
1588 struct amdgpu_winsys *ws = amdgpu_winsys(rws);
1589 amdgpu_bo_handle buf_handle;
1590 struct amdgpu_winsys_bo *bo;
1591 uint64_t va;
1592 amdgpu_va_handle va_handle;
1593 /* Avoid failure when the size is not page aligned */
1594 uint64_t aligned_size = align64(size, ws->info.gart_page_size);
1595
1596 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
1597 if (!bo)
1598 return NULL;
1599
1600 if (amdgpu_create_bo_from_user_mem(ws->dev, pointer,
1601 aligned_size, &buf_handle))
1602 goto error;
1603
1604 if (amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
1605 aligned_size,
1606 amdgpu_get_optimal_vm_alignment(ws, aligned_size,
1607 ws->info.gart_page_size),
1608 0, &va, &va_handle, AMDGPU_VA_RANGE_HIGH))
1609 goto error_va_alloc;
1610
1611 if (amdgpu_bo_va_op(buf_handle, 0, aligned_size, va, 0, AMDGPU_VA_OP_MAP))
1612 goto error_va_map;
1613
1614 /* Initialize it. */
1615 bo->is_user_ptr = true;
1616 pipe_reference_init(&bo->base.reference, 1);
1617 simple_mtx_init(&bo->lock, mtx_plain);
1618 bo->bo = buf_handle;
1619 bo->base.alignment = 0;
1620 bo->base.size = size;
1621 bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
1622 bo->ws = ws;
1623 bo->cpu_ptr = pointer;
1624 bo->va = va;
1625 bo->u.real.va_handle = va_handle;
1626 bo->initial_domain = RADEON_DOMAIN_GTT;
1627 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
1628
1629 ws->allocated_gtt += aligned_size;
1630
1631 amdgpu_add_buffer_to_global_list(bo);
1632
1633 amdgpu_bo_export(bo->bo, amdgpu_bo_handle_type_kms, &bo->u.real.kms_handle);
1634
1635 return (struct pb_buffer*)bo;
1636
1637 error_va_map:
1638 amdgpu_va_range_free(va_handle);
1639
1640 error_va_alloc:
1641 amdgpu_bo_free(buf_handle);
1642
1643 error:
1644 FREE(bo);
1645 return NULL;
1646 }
1647
1648 static bool amdgpu_bo_is_user_ptr(struct pb_buffer *buf)
1649 {
1650 return ((struct amdgpu_winsys_bo*)buf)->is_user_ptr;
1651 }
1652
1653 static bool amdgpu_bo_is_suballocated(struct pb_buffer *buf)
1654 {
1655 struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
1656
1657 return !bo->bo && !bo->sparse;
1658 }
1659
1660 static uint64_t amdgpu_bo_get_va(struct pb_buffer *buf)
1661 {
1662 return ((struct amdgpu_winsys_bo*)buf)->va;
1663 }
1664
1665 void amdgpu_bo_init_functions(struct amdgpu_screen_winsys *ws)
1666 {
1667 ws->base.buffer_set_metadata = amdgpu_buffer_set_metadata;
1668 ws->base.buffer_get_metadata = amdgpu_buffer_get_metadata;
1669 ws->base.buffer_map = amdgpu_bo_map;
1670 ws->base.buffer_unmap = amdgpu_bo_unmap;
1671 ws->base.buffer_wait = amdgpu_bo_wait;
1672 ws->base.buffer_create = amdgpu_buffer_create;
1673 ws->base.buffer_from_handle = amdgpu_bo_from_handle;
1674 ws->base.buffer_from_ptr = amdgpu_bo_from_ptr;
1675 ws->base.buffer_is_user_ptr = amdgpu_bo_is_user_ptr;
1676 ws->base.buffer_is_suballocated = amdgpu_bo_is_suballocated;
1677 ws->base.buffer_get_handle = amdgpu_bo_get_handle;
1678 ws->base.buffer_commit = amdgpu_bo_sparse_commit;
1679 ws->base.buffer_get_virtual_address = amdgpu_bo_get_va;
1680 ws->base.buffer_get_initial_domain = amdgpu_bo_get_initial_domain;
1681 ws->base.buffer_get_flags = amdgpu_bo_get_flags;
1682 }