winsys/amdgpu: sparse buffer debugging helpers
[mesa.git] / src / gallium / winsys / amdgpu / drm / amdgpu_bo.c
1 /*
2 * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
3 * Copyright © 2015 Advanced Micro Devices, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
16 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
18 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * The above copyright notice and this permission notice (including the
24 * next paragraph) shall be included in all copies or substantial portions
25 * of the Software.
26 */
27 /*
28 * Authors:
29 * Marek Olšák <maraeo@gmail.com>
30 */
31
32 #include "amdgpu_cs.h"
33
34 #include "os/os_time.h"
35 #include "state_tracker/drm_driver.h"
36 #include <amdgpu_drm.h>
37 #include <xf86drm.h>
38 #include <stdio.h>
39 #include <inttypes.h>
40
41 /* Set to 1 for verbose output showing committed sparse buffer ranges. */
42 #define DEBUG_SPARSE_COMMITS 0
43
44 struct amdgpu_sparse_backing_chunk {
45 uint32_t begin, end;
46 };
47
48 static struct pb_buffer *
49 amdgpu_bo_create(struct radeon_winsys *rws,
50 uint64_t size,
51 unsigned alignment,
52 enum radeon_bo_domain domain,
53 enum radeon_bo_flag flags);
54
55 static bool amdgpu_bo_wait(struct pb_buffer *_buf, uint64_t timeout,
56 enum radeon_bo_usage usage)
57 {
58 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
59 struct amdgpu_winsys *ws = bo->ws;
60 int64_t abs_timeout;
61
62 if (timeout == 0) {
63 if (p_atomic_read(&bo->num_active_ioctls))
64 return false;
65
66 } else {
67 abs_timeout = os_time_get_absolute_timeout(timeout);
68
69 /* Wait if any ioctl is being submitted with this buffer. */
70 if (!os_wait_until_zero_abs_timeout(&bo->num_active_ioctls, abs_timeout))
71 return false;
72 }
73
74 if (bo->is_shared) {
75 /* We can't use user fences for shared buffers, because user fences
76 * are local to this process only. If we want to wait for all buffer
77 * uses in all processes, we have to use amdgpu_bo_wait_for_idle.
78 */
79 bool buffer_busy = true;
80 int r;
81
82 r = amdgpu_bo_wait_for_idle(bo->bo, timeout, &buffer_busy);
83 if (r)
84 fprintf(stderr, "%s: amdgpu_bo_wait_for_idle failed %i\n", __func__,
85 r);
86 return !buffer_busy;
87 }
88
89 if (timeout == 0) {
90 unsigned idle_fences;
91 bool buffer_idle;
92
93 mtx_lock(&ws->bo_fence_lock);
94
95 for (idle_fences = 0; idle_fences < bo->num_fences; ++idle_fences) {
96 if (!amdgpu_fence_wait(bo->fences[idle_fences], 0, false))
97 break;
98 }
99
100 /* Release the idle fences to avoid checking them again later. */
101 for (unsigned i = 0; i < idle_fences; ++i)
102 amdgpu_fence_reference(&bo->fences[i], NULL);
103
104 memmove(&bo->fences[0], &bo->fences[idle_fences],
105 (bo->num_fences - idle_fences) * sizeof(*bo->fences));
106 bo->num_fences -= idle_fences;
107
108 buffer_idle = !bo->num_fences;
109 mtx_unlock(&ws->bo_fence_lock);
110
111 return buffer_idle;
112 } else {
113 bool buffer_idle = true;
114
115 mtx_lock(&ws->bo_fence_lock);
116 while (bo->num_fences && buffer_idle) {
117 struct pipe_fence_handle *fence = NULL;
118 bool fence_idle = false;
119
120 amdgpu_fence_reference(&fence, bo->fences[0]);
121
122 /* Wait for the fence. */
123 mtx_unlock(&ws->bo_fence_lock);
124 if (amdgpu_fence_wait(fence, abs_timeout, true))
125 fence_idle = true;
126 else
127 buffer_idle = false;
128 mtx_lock(&ws->bo_fence_lock);
129
130 /* Release an idle fence to avoid checking it again later, keeping in
131 * mind that the fence array may have been modified by other threads.
132 */
133 if (fence_idle && bo->num_fences && bo->fences[0] == fence) {
134 amdgpu_fence_reference(&bo->fences[0], NULL);
135 memmove(&bo->fences[0], &bo->fences[1],
136 (bo->num_fences - 1) * sizeof(*bo->fences));
137 bo->num_fences--;
138 }
139
140 amdgpu_fence_reference(&fence, NULL);
141 }
142 mtx_unlock(&ws->bo_fence_lock);
143
144 return buffer_idle;
145 }
146 }
147
148 static enum radeon_bo_domain amdgpu_bo_get_initial_domain(
149 struct pb_buffer *buf)
150 {
151 return ((struct amdgpu_winsys_bo*)buf)->initial_domain;
152 }
153
154 static void amdgpu_bo_remove_fences(struct amdgpu_winsys_bo *bo)
155 {
156 for (unsigned i = 0; i < bo->num_fences; ++i)
157 amdgpu_fence_reference(&bo->fences[i], NULL);
158
159 FREE(bo->fences);
160 bo->num_fences = 0;
161 bo->max_fences = 0;
162 }
163
164 void amdgpu_bo_destroy(struct pb_buffer *_buf)
165 {
166 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
167
168 assert(bo->bo && "must not be called for slab entries");
169
170 mtx_lock(&bo->ws->global_bo_list_lock);
171 LIST_DEL(&bo->u.real.global_list_item);
172 bo->ws->num_buffers--;
173 mtx_unlock(&bo->ws->global_bo_list_lock);
174
175 amdgpu_bo_va_op(bo->bo, 0, bo->base.size, bo->va, 0, AMDGPU_VA_OP_UNMAP);
176 amdgpu_va_range_free(bo->u.real.va_handle);
177 amdgpu_bo_free(bo->bo);
178
179 amdgpu_bo_remove_fences(bo);
180
181 if (bo->initial_domain & RADEON_DOMAIN_VRAM)
182 bo->ws->allocated_vram -= align64(bo->base.size, bo->ws->info.gart_page_size);
183 else if (bo->initial_domain & RADEON_DOMAIN_GTT)
184 bo->ws->allocated_gtt -= align64(bo->base.size, bo->ws->info.gart_page_size);
185
186 if (bo->u.real.map_count >= 1) {
187 if (bo->initial_domain & RADEON_DOMAIN_VRAM)
188 bo->ws->mapped_vram -= bo->base.size;
189 else if (bo->initial_domain & RADEON_DOMAIN_GTT)
190 bo->ws->mapped_gtt -= bo->base.size;
191 bo->ws->num_mapped_buffers--;
192 }
193
194 FREE(bo);
195 }
196
197 static void amdgpu_bo_destroy_or_cache(struct pb_buffer *_buf)
198 {
199 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
200
201 assert(bo->bo); /* slab buffers have a separate vtbl */
202
203 if (bo->u.real.use_reusable_pool)
204 pb_cache_add_buffer(&bo->u.real.cache_entry);
205 else
206 amdgpu_bo_destroy(_buf);
207 }
208
209 static void *amdgpu_bo_map(struct pb_buffer *buf,
210 struct radeon_winsys_cs *rcs,
211 enum pipe_transfer_usage usage)
212 {
213 struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
214 struct amdgpu_winsys_bo *real;
215 struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
216 int r;
217 void *cpu = NULL;
218 uint64_t offset = 0;
219
220 assert(!bo->sparse);
221
222 /* If it's not unsynchronized bo_map, flush CS if needed and then wait. */
223 if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
224 /* DONTBLOCK doesn't make sense with UNSYNCHRONIZED. */
225 if (usage & PIPE_TRANSFER_DONTBLOCK) {
226 if (!(usage & PIPE_TRANSFER_WRITE)) {
227 /* Mapping for read.
228 *
229 * Since we are mapping for read, we don't need to wait
230 * if the GPU is using the buffer for read too
231 * (neither one is changing it).
232 *
233 * Only check whether the buffer is being used for write. */
234 if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
235 RADEON_USAGE_WRITE)) {
236 cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
237 return NULL;
238 }
239
240 if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
241 RADEON_USAGE_WRITE)) {
242 return NULL;
243 }
244 } else {
245 if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo)) {
246 cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
247 return NULL;
248 }
249
250 if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
251 RADEON_USAGE_READWRITE)) {
252 return NULL;
253 }
254 }
255 } else {
256 uint64_t time = os_time_get_nano();
257
258 if (!(usage & PIPE_TRANSFER_WRITE)) {
259 /* Mapping for read.
260 *
261 * Since we are mapping for read, we don't need to wait
262 * if the GPU is using the buffer for read too
263 * (neither one is changing it).
264 *
265 * Only check whether the buffer is being used for write. */
266 if (cs) {
267 if (amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
268 RADEON_USAGE_WRITE)) {
269 cs->flush_cs(cs->flush_data, 0, NULL);
270 } else {
271 /* Try to avoid busy-waiting in amdgpu_bo_wait. */
272 if (p_atomic_read(&bo->num_active_ioctls))
273 amdgpu_cs_sync_flush(rcs);
274 }
275 }
276
277 amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
278 RADEON_USAGE_WRITE);
279 } else {
280 /* Mapping for write. */
281 if (cs) {
282 if (amdgpu_bo_is_referenced_by_cs(cs, bo)) {
283 cs->flush_cs(cs->flush_data, 0, NULL);
284 } else {
285 /* Try to avoid busy-waiting in amdgpu_bo_wait. */
286 if (p_atomic_read(&bo->num_active_ioctls))
287 amdgpu_cs_sync_flush(rcs);
288 }
289 }
290
291 amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
292 RADEON_USAGE_READWRITE);
293 }
294
295 bo->ws->buffer_wait_time += os_time_get_nano() - time;
296 }
297 }
298
299 /* If the buffer is created from user memory, return the user pointer. */
300 if (bo->user_ptr)
301 return bo->user_ptr;
302
303 if (bo->bo) {
304 real = bo;
305 } else {
306 real = bo->u.slab.real;
307 offset = bo->va - real->va;
308 }
309
310 r = amdgpu_bo_cpu_map(real->bo, &cpu);
311 if (r) {
312 /* Clear the cache and try again. */
313 pb_cache_release_all_buffers(&real->ws->bo_cache);
314 r = amdgpu_bo_cpu_map(real->bo, &cpu);
315 if (r)
316 return NULL;
317 }
318
319 if (p_atomic_inc_return(&real->u.real.map_count) == 1) {
320 if (real->initial_domain & RADEON_DOMAIN_VRAM)
321 real->ws->mapped_vram += real->base.size;
322 else if (real->initial_domain & RADEON_DOMAIN_GTT)
323 real->ws->mapped_gtt += real->base.size;
324 real->ws->num_mapped_buffers++;
325 }
326 return (uint8_t*)cpu + offset;
327 }
328
329 static void amdgpu_bo_unmap(struct pb_buffer *buf)
330 {
331 struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
332 struct amdgpu_winsys_bo *real;
333
334 assert(!bo->sparse);
335
336 if (bo->user_ptr)
337 return;
338
339 real = bo->bo ? bo : bo->u.slab.real;
340
341 if (p_atomic_dec_zero(&real->u.real.map_count)) {
342 if (real->initial_domain & RADEON_DOMAIN_VRAM)
343 real->ws->mapped_vram -= real->base.size;
344 else if (real->initial_domain & RADEON_DOMAIN_GTT)
345 real->ws->mapped_gtt -= real->base.size;
346 real->ws->num_mapped_buffers--;
347 }
348
349 amdgpu_bo_cpu_unmap(real->bo);
350 }
351
352 static const struct pb_vtbl amdgpu_winsys_bo_vtbl = {
353 amdgpu_bo_destroy_or_cache
354 /* other functions are never called */
355 };
356
357 static void amdgpu_add_buffer_to_global_list(struct amdgpu_winsys_bo *bo)
358 {
359 struct amdgpu_winsys *ws = bo->ws;
360
361 assert(bo->bo);
362
363 mtx_lock(&ws->global_bo_list_lock);
364 LIST_ADDTAIL(&bo->u.real.global_list_item, &ws->global_bo_list);
365 ws->num_buffers++;
366 mtx_unlock(&ws->global_bo_list_lock);
367 }
368
369 static struct amdgpu_winsys_bo *amdgpu_create_bo(struct amdgpu_winsys *ws,
370 uint64_t size,
371 unsigned alignment,
372 unsigned usage,
373 enum radeon_bo_domain initial_domain,
374 unsigned flags,
375 unsigned pb_cache_bucket)
376 {
377 struct amdgpu_bo_alloc_request request = {0};
378 amdgpu_bo_handle buf_handle;
379 uint64_t va = 0;
380 struct amdgpu_winsys_bo *bo;
381 amdgpu_va_handle va_handle;
382 unsigned va_gap_size;
383 int r;
384
385 assert(initial_domain & RADEON_DOMAIN_VRAM_GTT);
386 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
387 if (!bo) {
388 return NULL;
389 }
390
391 pb_cache_init_entry(&ws->bo_cache, &bo->u.real.cache_entry, &bo->base,
392 pb_cache_bucket);
393 request.alloc_size = size;
394 request.phys_alignment = alignment;
395
396 if (initial_domain & RADEON_DOMAIN_VRAM)
397 request.preferred_heap |= AMDGPU_GEM_DOMAIN_VRAM;
398 if (initial_domain & RADEON_DOMAIN_GTT)
399 request.preferred_heap |= AMDGPU_GEM_DOMAIN_GTT;
400
401 if (flags & RADEON_FLAG_CPU_ACCESS)
402 request.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
403 if (flags & RADEON_FLAG_NO_CPU_ACCESS)
404 request.flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
405 if (flags & RADEON_FLAG_GTT_WC)
406 request.flags |= AMDGPU_GEM_CREATE_CPU_GTT_USWC;
407
408 r = amdgpu_bo_alloc(ws->dev, &request, &buf_handle);
409 if (r) {
410 fprintf(stderr, "amdgpu: Failed to allocate a buffer:\n");
411 fprintf(stderr, "amdgpu: size : %"PRIu64" bytes\n", size);
412 fprintf(stderr, "amdgpu: alignment : %u bytes\n", alignment);
413 fprintf(stderr, "amdgpu: domains : %u\n", initial_domain);
414 goto error_bo_alloc;
415 }
416
417 va_gap_size = ws->check_vm ? MAX2(4 * alignment, 64 * 1024) : 0;
418 r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
419 size + va_gap_size, alignment, 0, &va, &va_handle, 0);
420 if (r)
421 goto error_va_alloc;
422
423 r = amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP);
424 if (r)
425 goto error_va_map;
426
427 pipe_reference_init(&bo->base.reference, 1);
428 bo->base.alignment = alignment;
429 bo->base.usage = usage;
430 bo->base.size = size;
431 bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
432 bo->ws = ws;
433 bo->bo = buf_handle;
434 bo->va = va;
435 bo->u.real.va_handle = va_handle;
436 bo->initial_domain = initial_domain;
437 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
438
439 if (initial_domain & RADEON_DOMAIN_VRAM)
440 ws->allocated_vram += align64(size, ws->info.gart_page_size);
441 else if (initial_domain & RADEON_DOMAIN_GTT)
442 ws->allocated_gtt += align64(size, ws->info.gart_page_size);
443
444 amdgpu_add_buffer_to_global_list(bo);
445
446 return bo;
447
448 error_va_map:
449 amdgpu_va_range_free(va_handle);
450
451 error_va_alloc:
452 amdgpu_bo_free(buf_handle);
453
454 error_bo_alloc:
455 FREE(bo);
456 return NULL;
457 }
458
459 bool amdgpu_bo_can_reclaim(struct pb_buffer *_buf)
460 {
461 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
462
463 if (amdgpu_bo_is_referenced_by_any_cs(bo)) {
464 return false;
465 }
466
467 return amdgpu_bo_wait(_buf, 0, RADEON_USAGE_READWRITE);
468 }
469
470 bool amdgpu_bo_can_reclaim_slab(void *priv, struct pb_slab_entry *entry)
471 {
472 struct amdgpu_winsys_bo *bo = NULL; /* fix container_of */
473 bo = container_of(entry, bo, u.slab.entry);
474
475 return amdgpu_bo_can_reclaim(&bo->base);
476 }
477
478 static void amdgpu_bo_slab_destroy(struct pb_buffer *_buf)
479 {
480 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
481
482 assert(!bo->bo);
483
484 pb_slab_free(&bo->ws->bo_slabs, &bo->u.slab.entry);
485 }
486
487 static const struct pb_vtbl amdgpu_winsys_bo_slab_vtbl = {
488 amdgpu_bo_slab_destroy
489 /* other functions are never called */
490 };
491
492 struct pb_slab *amdgpu_bo_slab_alloc(void *priv, unsigned heap,
493 unsigned entry_size,
494 unsigned group_index)
495 {
496 struct amdgpu_winsys *ws = priv;
497 struct amdgpu_slab *slab = CALLOC_STRUCT(amdgpu_slab);
498 enum radeon_bo_domain domains;
499 enum radeon_bo_flag flags = 0;
500 uint32_t base_id;
501
502 if (!slab)
503 return NULL;
504
505 if (heap & 1)
506 flags |= RADEON_FLAG_GTT_WC;
507 if (heap & 2)
508 flags |= RADEON_FLAG_CPU_ACCESS;
509
510 switch (heap >> 2) {
511 case 0:
512 domains = RADEON_DOMAIN_VRAM;
513 break;
514 default:
515 case 1:
516 domains = RADEON_DOMAIN_VRAM_GTT;
517 break;
518 case 2:
519 domains = RADEON_DOMAIN_GTT;
520 break;
521 }
522
523 slab->buffer = amdgpu_winsys_bo(amdgpu_bo_create(&ws->base,
524 64 * 1024, 64 * 1024,
525 domains, flags));
526 if (!slab->buffer)
527 goto fail;
528
529 assert(slab->buffer->bo);
530
531 slab->base.num_entries = slab->buffer->base.size / entry_size;
532 slab->base.num_free = slab->base.num_entries;
533 slab->entries = CALLOC(slab->base.num_entries, sizeof(*slab->entries));
534 if (!slab->entries)
535 goto fail_buffer;
536
537 LIST_INITHEAD(&slab->base.free);
538
539 base_id = __sync_fetch_and_add(&ws->next_bo_unique_id, slab->base.num_entries);
540
541 for (unsigned i = 0; i < slab->base.num_entries; ++i) {
542 struct amdgpu_winsys_bo *bo = &slab->entries[i];
543
544 bo->base.alignment = entry_size;
545 bo->base.usage = slab->buffer->base.usage;
546 bo->base.size = entry_size;
547 bo->base.vtbl = &amdgpu_winsys_bo_slab_vtbl;
548 bo->ws = ws;
549 bo->va = slab->buffer->va + i * entry_size;
550 bo->initial_domain = domains;
551 bo->unique_id = base_id + i;
552 bo->u.slab.entry.slab = &slab->base;
553 bo->u.slab.entry.group_index = group_index;
554 bo->u.slab.real = slab->buffer;
555
556 LIST_ADDTAIL(&bo->u.slab.entry.head, &slab->base.free);
557 }
558
559 return &slab->base;
560
561 fail_buffer:
562 amdgpu_winsys_bo_reference(&slab->buffer, NULL);
563 fail:
564 FREE(slab);
565 return NULL;
566 }
567
568 void amdgpu_bo_slab_free(void *priv, struct pb_slab *pslab)
569 {
570 struct amdgpu_slab *slab = amdgpu_slab(pslab);
571
572 for (unsigned i = 0; i < slab->base.num_entries; ++i)
573 amdgpu_bo_remove_fences(&slab->entries[i]);
574
575 FREE(slab->entries);
576 amdgpu_winsys_bo_reference(&slab->buffer, NULL);
577 FREE(slab);
578 }
579
580 #if DEBUG_SPARSE_COMMITS
581 static void
582 sparse_dump(struct amdgpu_winsys_bo *bo, const char *func)
583 {
584 fprintf(stderr, "%s: %p (size=%"PRIu64", num_va_pages=%u) @ %s\n"
585 "Commitments:\n",
586 __func__, bo, bo->base.size, bo->u.sparse.num_va_pages, func);
587
588 struct amdgpu_sparse_backing *span_backing = NULL;
589 uint32_t span_first_backing_page = 0;
590 uint32_t span_first_va_page = 0;
591 uint32_t va_page = 0;
592
593 for (;;) {
594 struct amdgpu_sparse_backing *backing = 0;
595 uint32_t backing_page = 0;
596
597 if (va_page < bo->u.sparse.num_va_pages) {
598 backing = bo->u.sparse.commitments[va_page].backing;
599 backing_page = bo->u.sparse.commitments[va_page].page;
600 }
601
602 if (span_backing &&
603 (backing != span_backing ||
604 backing_page != span_first_backing_page + (va_page - span_first_va_page))) {
605 fprintf(stderr, " %u..%u: backing=%p:%u..%u\n",
606 span_first_va_page, va_page - 1, span_backing,
607 span_first_backing_page,
608 span_first_backing_page + (va_page - span_first_va_page) - 1);
609
610 span_backing = NULL;
611 }
612
613 if (va_page >= bo->u.sparse.num_va_pages)
614 break;
615
616 if (backing && !span_backing) {
617 span_backing = backing;
618 span_first_backing_page = backing_page;
619 span_first_va_page = va_page;
620 }
621
622 va_page++;
623 }
624
625 fprintf(stderr, "Backing:\n");
626
627 list_for_each_entry(struct amdgpu_sparse_backing, backing, &bo->u.sparse.backing, list) {
628 fprintf(stderr, " %p (size=%"PRIu64")\n", backing, backing->bo->base.size);
629 for (unsigned i = 0; i < backing->num_chunks; ++i)
630 fprintf(stderr, " %u..%u\n", backing->chunks[i].begin, backing->chunks[i].end);
631 }
632 }
633 #endif
634
635 /*
636 * Attempt to allocate the given number of backing pages. Fewer pages may be
637 * allocated (depending on the fragmentation of existing backing buffers),
638 * which will be reflected by a change to *pnum_pages.
639 */
640 static struct amdgpu_sparse_backing *
641 sparse_backing_alloc(struct amdgpu_winsys_bo *bo, uint32_t *pstart_page, uint32_t *pnum_pages)
642 {
643 struct amdgpu_sparse_backing *best_backing;
644 unsigned best_idx;
645 uint32_t best_num_pages;
646
647 best_backing = NULL;
648 best_idx = 0;
649 best_num_pages = 0;
650
651 /* This is a very simple and inefficient best-fit algorithm. */
652 list_for_each_entry(struct amdgpu_sparse_backing, backing, &bo->u.sparse.backing, list) {
653 for (unsigned idx = 0; idx < backing->num_chunks; ++idx) {
654 uint32_t cur_num_pages = backing->chunks[idx].end - backing->chunks[idx].begin;
655 if ((best_num_pages < *pnum_pages && cur_num_pages > best_num_pages) ||
656 (best_num_pages > *pnum_pages && cur_num_pages < best_num_pages)) {
657 best_backing = backing;
658 best_idx = idx;
659 best_num_pages = cur_num_pages;
660 }
661 }
662 }
663
664 /* Allocate a new backing buffer if necessary. */
665 if (!best_backing) {
666 struct pb_buffer *buf;
667 uint64_t size;
668 uint32_t pages;
669
670 best_backing = CALLOC_STRUCT(amdgpu_sparse_backing);
671 if (!best_backing)
672 return NULL;
673
674 best_backing->max_chunks = 4;
675 best_backing->chunks = CALLOC(best_backing->max_chunks,
676 sizeof(*best_backing->chunks));
677 if (!best_backing->chunks) {
678 FREE(best_backing);
679 return NULL;
680 }
681
682 assert(bo->u.sparse.num_backing_pages < DIV_ROUND_UP(bo->base.size, RADEON_SPARSE_PAGE_SIZE));
683
684 size = MIN3(bo->base.size / 16,
685 8 * 1024 * 1024,
686 bo->base.size - (uint64_t)bo->u.sparse.num_backing_pages * RADEON_SPARSE_PAGE_SIZE);
687 size = MAX2(size, RADEON_SPARSE_PAGE_SIZE);
688
689 buf = amdgpu_bo_create(&bo->ws->base, size, RADEON_SPARSE_PAGE_SIZE,
690 bo->initial_domain,
691 bo->u.sparse.flags | RADEON_FLAG_HANDLE);
692 if (!buf) {
693 FREE(best_backing->chunks);
694 FREE(best_backing);
695 return NULL;
696 }
697
698 /* We might have gotten a bigger buffer than requested via caching. */
699 pages = buf->size / RADEON_SPARSE_PAGE_SIZE;
700
701 best_backing->bo = amdgpu_winsys_bo(buf);
702 best_backing->num_chunks = 1;
703 best_backing->chunks[0].begin = 0;
704 best_backing->chunks[0].end = pages;
705
706 list_add(&best_backing->list, &bo->u.sparse.backing);
707 bo->u.sparse.num_backing_pages += pages;
708
709 best_idx = 0;
710 best_num_pages = pages;
711 }
712
713 *pnum_pages = MIN2(*pnum_pages, best_num_pages);
714 *pstart_page = best_backing->chunks[best_idx].begin;
715 best_backing->chunks[best_idx].begin += *pnum_pages;
716
717 if (best_backing->chunks[best_idx].begin >= best_backing->chunks[best_idx].end) {
718 memmove(&best_backing->chunks[best_idx], &best_backing->chunks[best_idx + 1],
719 sizeof(*best_backing->chunks) * (best_backing->num_chunks - best_idx - 1));
720 best_backing->num_chunks--;
721 }
722
723 return best_backing;
724 }
725
726 static void
727 sparse_free_backing_buffer(struct amdgpu_winsys_bo *bo,
728 struct amdgpu_sparse_backing *backing)
729 {
730 struct amdgpu_winsys *ws = backing->bo->ws;
731
732 bo->u.sparse.num_backing_pages -= backing->bo->base.size / RADEON_SPARSE_PAGE_SIZE;
733
734 mtx_lock(&ws->bo_fence_lock);
735 amdgpu_add_fences(backing->bo, bo->num_fences, bo->fences);
736 mtx_unlock(&ws->bo_fence_lock);
737
738 list_del(&backing->list);
739 amdgpu_winsys_bo_reference(&backing->bo, NULL);
740 FREE(backing->chunks);
741 FREE(backing);
742 }
743
744 /*
745 * Return a range of pages from the given backing buffer back into the
746 * free structure.
747 */
748 static bool
749 sparse_backing_free(struct amdgpu_winsys_bo *bo,
750 struct amdgpu_sparse_backing *backing,
751 uint32_t start_page, uint32_t num_pages)
752 {
753 uint32_t end_page = start_page + num_pages;
754 unsigned low = 0;
755 unsigned high = backing->num_chunks;
756
757 /* Find the first chunk with begin >= start_page. */
758 while (low < high) {
759 unsigned mid = low + (high - low) / 2;
760
761 if (backing->chunks[mid].begin >= start_page)
762 high = mid;
763 else
764 low = mid + 1;
765 }
766
767 assert(low >= backing->num_chunks || end_page <= backing->chunks[low].begin);
768 assert(low == 0 || backing->chunks[low - 1].end <= start_page);
769
770 if (low > 0 && backing->chunks[low - 1].end == start_page) {
771 backing->chunks[low - 1].end = end_page;
772
773 if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
774 backing->chunks[low - 1].end = backing->chunks[low].end;
775 memmove(&backing->chunks[low], &backing->chunks[low + 1],
776 sizeof(*backing->chunks) * (backing->num_chunks - low - 1));
777 backing->num_chunks--;
778 }
779 } else if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
780 backing->chunks[low].begin = start_page;
781 } else {
782 if (backing->num_chunks >= backing->max_chunks) {
783 unsigned new_max_chunks = 2 * backing->max_chunks;
784 struct amdgpu_sparse_backing_chunk *new_chunks =
785 REALLOC(backing->chunks,
786 sizeof(*backing->chunks) * backing->max_chunks,
787 sizeof(*backing->chunks) * new_max_chunks);
788 if (!new_chunks)
789 return false;
790
791 backing->max_chunks = new_max_chunks;
792 backing->chunks = new_chunks;
793 }
794
795 memmove(&backing->chunks[low + 1], &backing->chunks[low],
796 sizeof(*backing->chunks) * (backing->num_chunks - low));
797 backing->chunks[low].begin = start_page;
798 backing->chunks[low].end = end_page;
799 backing->num_chunks++;
800 }
801
802 if (backing->num_chunks == 1 && backing->chunks[0].begin == 0 &&
803 backing->chunks[0].end == backing->bo->base.size / RADEON_SPARSE_PAGE_SIZE)
804 sparse_free_backing_buffer(bo, backing);
805
806 return true;
807 }
808
809 static void amdgpu_bo_sparse_destroy(struct pb_buffer *_buf)
810 {
811 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
812 int r;
813
814 assert(!bo->bo && bo->sparse);
815
816 r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0,
817 (uint64_t)bo->u.sparse.num_va_pages * RADEON_SPARSE_PAGE_SIZE,
818 bo->va, 0, AMDGPU_VA_OP_CLEAR);
819 if (r) {
820 fprintf(stderr, "amdgpu: clearing PRT VA region on destroy failed (%d)\n", r);
821 }
822
823 while (!list_empty(&bo->u.sparse.backing)) {
824 struct amdgpu_sparse_backing *dummy = NULL;
825 sparse_free_backing_buffer(bo,
826 container_of(bo->u.sparse.backing.next,
827 dummy, list));
828 }
829
830 amdgpu_va_range_free(bo->u.sparse.va_handle);
831 mtx_destroy(&bo->u.sparse.commit_lock);
832 FREE(bo->u.sparse.commitments);
833 FREE(bo);
834 }
835
836 static const struct pb_vtbl amdgpu_winsys_bo_sparse_vtbl = {
837 amdgpu_bo_sparse_destroy
838 /* other functions are never called */
839 };
840
841 static struct pb_buffer *
842 amdgpu_bo_sparse_create(struct amdgpu_winsys *ws, uint64_t size,
843 enum radeon_bo_domain domain,
844 enum radeon_bo_flag flags)
845 {
846 struct amdgpu_winsys_bo *bo;
847 uint64_t map_size;
848 uint64_t va_gap_size;
849 int r;
850
851 /* We use 32-bit page numbers; refuse to attempt allocating sparse buffers
852 * that exceed this limit. This is not really a restriction: we don't have
853 * that much virtual address space anyway.
854 */
855 if (size > (uint64_t)INT32_MAX * RADEON_SPARSE_PAGE_SIZE)
856 return NULL;
857
858 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
859 if (!bo)
860 return NULL;
861
862 pipe_reference_init(&bo->base.reference, 1);
863 bo->base.alignment = RADEON_SPARSE_PAGE_SIZE;
864 bo->base.size = size;
865 bo->base.vtbl = &amdgpu_winsys_bo_sparse_vtbl;
866 bo->ws = ws;
867 bo->initial_domain = domain;
868 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
869 bo->sparse = true;
870 bo->u.sparse.flags = flags & ~RADEON_FLAG_SPARSE;
871
872 bo->u.sparse.num_va_pages = DIV_ROUND_UP(size, RADEON_SPARSE_PAGE_SIZE);
873 bo->u.sparse.commitments = CALLOC(bo->u.sparse.num_va_pages,
874 sizeof(*bo->u.sparse.commitments));
875 if (!bo->u.sparse.commitments)
876 goto error_alloc_commitments;
877
878 mtx_init(&bo->u.sparse.commit_lock, mtx_plain);
879 LIST_INITHEAD(&bo->u.sparse.backing);
880
881 /* For simplicity, we always map a multiple of the page size. */
882 map_size = align64(size, RADEON_SPARSE_PAGE_SIZE);
883 va_gap_size = ws->check_vm ? 4 * RADEON_SPARSE_PAGE_SIZE : 0;
884 r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
885 map_size + va_gap_size, RADEON_SPARSE_PAGE_SIZE,
886 0, &bo->va, &bo->u.sparse.va_handle, 0);
887 if (r)
888 goto error_va_alloc;
889
890 r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0, size, bo->va,
891 AMDGPU_VM_PAGE_PRT, AMDGPU_VA_OP_MAP);
892 if (r)
893 goto error_va_map;
894
895 return &bo->base;
896
897 error_va_map:
898 amdgpu_va_range_free(bo->u.sparse.va_handle);
899 error_va_alloc:
900 mtx_destroy(&bo->u.sparse.commit_lock);
901 FREE(bo->u.sparse.commitments);
902 error_alloc_commitments:
903 FREE(bo);
904 return NULL;
905 }
906
907 static bool
908 amdgpu_bo_sparse_commit(struct pb_buffer *buf, uint64_t offset, uint64_t size,
909 bool commit)
910 {
911 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(buf);
912 struct amdgpu_sparse_commitment *comm;
913 uint32_t va_page, end_va_page;
914 bool ok = true;
915 int r;
916
917 assert(bo->sparse);
918 assert(offset % RADEON_SPARSE_PAGE_SIZE == 0);
919 assert(offset <= bo->base.size);
920 assert(size <= bo->base.size - offset);
921 assert(size % RADEON_SPARSE_PAGE_SIZE == 0 || offset + size == bo->base.size);
922
923 comm = bo->u.sparse.commitments;
924 va_page = offset / RADEON_SPARSE_PAGE_SIZE;
925 end_va_page = va_page + DIV_ROUND_UP(size, RADEON_SPARSE_PAGE_SIZE);
926
927 mtx_lock(&bo->u.sparse.commit_lock);
928
929 #if DEBUG_SPARSE_COMMITS
930 sparse_dump(bo, __func__);
931 #endif
932
933 if (commit) {
934 while (va_page < end_va_page) {
935 uint32_t span_va_page;
936
937 /* Skip pages that are already committed. */
938 if (comm[va_page].backing) {
939 va_page++;
940 continue;
941 }
942
943 /* Determine length of uncommitted span. */
944 span_va_page = va_page;
945 while (va_page < end_va_page && !comm[va_page].backing)
946 va_page++;
947
948 /* Fill the uncommitted span with chunks of backing memory. */
949 while (span_va_page < va_page) {
950 struct amdgpu_sparse_backing *backing;
951 uint32_t backing_start, backing_size;
952
953 backing_size = va_page - span_va_page;
954 backing = sparse_backing_alloc(bo, &backing_start, &backing_size);
955 if (!backing) {
956 ok = false;
957 goto out;
958 }
959
960 r = amdgpu_bo_va_op_raw(bo->ws->dev, backing->bo->bo,
961 (uint64_t)backing_start * RADEON_SPARSE_PAGE_SIZE,
962 (uint64_t)backing_size * RADEON_SPARSE_PAGE_SIZE,
963 bo->va + (uint64_t)span_va_page * RADEON_SPARSE_PAGE_SIZE,
964 AMDGPU_VM_PAGE_READABLE |
965 AMDGPU_VM_PAGE_WRITEABLE |
966 AMDGPU_VM_PAGE_EXECUTABLE,
967 AMDGPU_VA_OP_REPLACE);
968 if (r) {
969 ok = sparse_backing_free(bo, backing, backing_start, backing_size);
970 assert(ok && "sufficient memory should already be allocated");
971
972 ok = false;
973 goto out;
974 }
975
976 while (backing_size) {
977 comm[span_va_page].backing = backing;
978 comm[span_va_page].page = backing_start;
979 span_va_page++;
980 backing_start++;
981 backing_size--;
982 }
983 }
984 }
985 } else {
986 r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0,
987 (uint64_t)(end_va_page - va_page) * RADEON_SPARSE_PAGE_SIZE,
988 bo->va + (uint64_t)va_page * RADEON_SPARSE_PAGE_SIZE,
989 AMDGPU_VM_PAGE_PRT, AMDGPU_VA_OP_REPLACE);
990 if (r) {
991 ok = false;
992 goto out;
993 }
994
995 while (va_page < end_va_page) {
996 struct amdgpu_sparse_backing *backing;
997 uint32_t backing_start;
998 uint32_t span_pages;
999
1000 /* Skip pages that are already uncommitted. */
1001 if (!comm[va_page].backing) {
1002 va_page++;
1003 continue;
1004 }
1005
1006 /* Group contiguous spans of pages. */
1007 backing = comm[va_page].backing;
1008 backing_start = comm[va_page].page;
1009 comm[va_page].backing = NULL;
1010
1011 span_pages = 1;
1012 va_page++;
1013
1014 while (va_page < end_va_page &&
1015 comm[va_page].backing == backing &&
1016 comm[va_page].page == backing_start + span_pages) {
1017 comm[va_page].backing = NULL;
1018 va_page++;
1019 span_pages++;
1020 }
1021
1022 if (!sparse_backing_free(bo, backing, backing_start, span_pages)) {
1023 /* Couldn't allocate tracking data structures, so we have to leak */
1024 fprintf(stderr, "amdgpu: leaking PRT backing memory\n");
1025 ok = false;
1026 }
1027 }
1028 }
1029 out:
1030
1031 mtx_unlock(&bo->u.sparse.commit_lock);
1032
1033 return ok;
1034 }
1035
1036 static unsigned eg_tile_split(unsigned tile_split)
1037 {
1038 switch (tile_split) {
1039 case 0: tile_split = 64; break;
1040 case 1: tile_split = 128; break;
1041 case 2: tile_split = 256; break;
1042 case 3: tile_split = 512; break;
1043 default:
1044 case 4: tile_split = 1024; break;
1045 case 5: tile_split = 2048; break;
1046 case 6: tile_split = 4096; break;
1047 }
1048 return tile_split;
1049 }
1050
1051 static unsigned eg_tile_split_rev(unsigned eg_tile_split)
1052 {
1053 switch (eg_tile_split) {
1054 case 64: return 0;
1055 case 128: return 1;
1056 case 256: return 2;
1057 case 512: return 3;
1058 default:
1059 case 1024: return 4;
1060 case 2048: return 5;
1061 case 4096: return 6;
1062 }
1063 }
1064
1065 static void amdgpu_buffer_get_metadata(struct pb_buffer *_buf,
1066 struct radeon_bo_metadata *md)
1067 {
1068 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
1069 struct amdgpu_bo_info info = {0};
1070 uint64_t tiling_flags;
1071 int r;
1072
1073 assert(bo->bo && "must not be called for slab entries");
1074
1075 r = amdgpu_bo_query_info(bo->bo, &info);
1076 if (r)
1077 return;
1078
1079 tiling_flags = info.metadata.tiling_info;
1080
1081 if (bo->ws->info.chip_class >= GFX9) {
1082 md->u.gfx9.swizzle_mode = AMDGPU_TILING_GET(tiling_flags, SWIZZLE_MODE);
1083 } else {
1084 md->u.legacy.microtile = RADEON_LAYOUT_LINEAR;
1085 md->u.legacy.macrotile = RADEON_LAYOUT_LINEAR;
1086
1087 if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 4) /* 2D_TILED_THIN1 */
1088 md->u.legacy.macrotile = RADEON_LAYOUT_TILED;
1089 else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 2) /* 1D_TILED_THIN1 */
1090 md->u.legacy.microtile = RADEON_LAYOUT_TILED;
1091
1092 md->u.legacy.pipe_config = AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG);
1093 md->u.legacy.bankw = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_WIDTH);
1094 md->u.legacy.bankh = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_HEIGHT);
1095 md->u.legacy.tile_split = eg_tile_split(AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT));
1096 md->u.legacy.mtilea = 1 << AMDGPU_TILING_GET(tiling_flags, MACRO_TILE_ASPECT);
1097 md->u.legacy.num_banks = 2 << AMDGPU_TILING_GET(tiling_flags, NUM_BANKS);
1098 md->u.legacy.scanout = AMDGPU_TILING_GET(tiling_flags, MICRO_TILE_MODE) == 0; /* DISPLAY */
1099 }
1100
1101 md->size_metadata = info.metadata.size_metadata;
1102 memcpy(md->metadata, info.metadata.umd_metadata, sizeof(md->metadata));
1103 }
1104
1105 static void amdgpu_buffer_set_metadata(struct pb_buffer *_buf,
1106 struct radeon_bo_metadata *md)
1107 {
1108 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
1109 struct amdgpu_bo_metadata metadata = {0};
1110 uint64_t tiling_flags = 0;
1111
1112 assert(bo->bo && "must not be called for slab entries");
1113
1114 if (bo->ws->info.chip_class >= GFX9) {
1115 tiling_flags |= AMDGPU_TILING_SET(SWIZZLE_MODE, md->u.gfx9.swizzle_mode);
1116 } else {
1117 if (md->u.legacy.macrotile == RADEON_LAYOUT_TILED)
1118 tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 4); /* 2D_TILED_THIN1 */
1119 else if (md->u.legacy.microtile == RADEON_LAYOUT_TILED)
1120 tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 2); /* 1D_TILED_THIN1 */
1121 else
1122 tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 1); /* LINEAR_ALIGNED */
1123
1124 tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, md->u.legacy.pipe_config);
1125 tiling_flags |= AMDGPU_TILING_SET(BANK_WIDTH, util_logbase2(md->u.legacy.bankw));
1126 tiling_flags |= AMDGPU_TILING_SET(BANK_HEIGHT, util_logbase2(md->u.legacy.bankh));
1127 if (md->u.legacy.tile_split)
1128 tiling_flags |= AMDGPU_TILING_SET(TILE_SPLIT, eg_tile_split_rev(md->u.legacy.tile_split));
1129 tiling_flags |= AMDGPU_TILING_SET(MACRO_TILE_ASPECT, util_logbase2(md->u.legacy.mtilea));
1130 tiling_flags |= AMDGPU_TILING_SET(NUM_BANKS, util_logbase2(md->u.legacy.num_banks)-1);
1131
1132 if (md->u.legacy.scanout)
1133 tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 0); /* DISPLAY_MICRO_TILING */
1134 else
1135 tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 1); /* THIN_MICRO_TILING */
1136 }
1137
1138 metadata.tiling_info = tiling_flags;
1139 metadata.size_metadata = md->size_metadata;
1140 memcpy(metadata.umd_metadata, md->metadata, sizeof(md->metadata));
1141
1142 amdgpu_bo_set_metadata(bo->bo, &metadata);
1143 }
1144
1145 static struct pb_buffer *
1146 amdgpu_bo_create(struct radeon_winsys *rws,
1147 uint64_t size,
1148 unsigned alignment,
1149 enum radeon_bo_domain domain,
1150 enum radeon_bo_flag flags)
1151 {
1152 struct amdgpu_winsys *ws = amdgpu_winsys(rws);
1153 struct amdgpu_winsys_bo *bo;
1154 unsigned usage = 0, pb_cache_bucket;
1155
1156 /* Sub-allocate small buffers from slabs. */
1157 if (!(flags & (RADEON_FLAG_HANDLE | RADEON_FLAG_SPARSE)) &&
1158 size <= (1 << AMDGPU_SLAB_MAX_SIZE_LOG2) &&
1159 alignment <= MAX2(1 << AMDGPU_SLAB_MIN_SIZE_LOG2, util_next_power_of_two(size))) {
1160 struct pb_slab_entry *entry;
1161 unsigned heap = 0;
1162
1163 if (flags & RADEON_FLAG_GTT_WC)
1164 heap |= 1;
1165 if (flags & RADEON_FLAG_CPU_ACCESS)
1166 heap |= 2;
1167 if (flags & ~(RADEON_FLAG_GTT_WC | RADEON_FLAG_CPU_ACCESS))
1168 goto no_slab;
1169
1170 switch (domain) {
1171 case RADEON_DOMAIN_VRAM:
1172 heap |= 0 * 4;
1173 break;
1174 case RADEON_DOMAIN_VRAM_GTT:
1175 heap |= 1 * 4;
1176 break;
1177 case RADEON_DOMAIN_GTT:
1178 heap |= 2 * 4;
1179 break;
1180 default:
1181 goto no_slab;
1182 }
1183
1184 entry = pb_slab_alloc(&ws->bo_slabs, size, heap);
1185 if (!entry) {
1186 /* Clear the cache and try again. */
1187 pb_cache_release_all_buffers(&ws->bo_cache);
1188
1189 entry = pb_slab_alloc(&ws->bo_slabs, size, heap);
1190 }
1191 if (!entry)
1192 return NULL;
1193
1194 bo = NULL;
1195 bo = container_of(entry, bo, u.slab.entry);
1196
1197 pipe_reference_init(&bo->base.reference, 1);
1198
1199 return &bo->base;
1200 }
1201 no_slab:
1202
1203 if (flags & RADEON_FLAG_SPARSE) {
1204 assert(RADEON_SPARSE_PAGE_SIZE % alignment == 0);
1205 assert(!(flags & RADEON_FLAG_CPU_ACCESS));
1206
1207 flags |= RADEON_FLAG_NO_CPU_ACCESS;
1208
1209 return amdgpu_bo_sparse_create(ws, size, domain, flags);
1210 }
1211
1212 /* This flag is irrelevant for the cache. */
1213 flags &= ~RADEON_FLAG_HANDLE;
1214
1215 /* Align size to page size. This is the minimum alignment for normal
1216 * BOs. Aligning this here helps the cached bufmgr. Especially small BOs,
1217 * like constant/uniform buffers, can benefit from better and more reuse.
1218 */
1219 size = align64(size, ws->info.gart_page_size);
1220 alignment = align(alignment, ws->info.gart_page_size);
1221
1222 /* Only set one usage bit each for domains and flags, or the cache manager
1223 * might consider different sets of domains / flags compatible
1224 */
1225 if (domain == RADEON_DOMAIN_VRAM_GTT)
1226 usage = 1 << 2;
1227 else
1228 usage = domain >> 1;
1229 assert(flags < sizeof(usage) * 8 - 3);
1230 usage |= 1 << (flags + 3);
1231
1232 /* Determine the pb_cache bucket for minimizing pb_cache misses. */
1233 pb_cache_bucket = 0;
1234 if (domain & RADEON_DOMAIN_VRAM) /* VRAM or VRAM+GTT */
1235 pb_cache_bucket += 1;
1236 if (flags == RADEON_FLAG_GTT_WC) /* WC */
1237 pb_cache_bucket += 2;
1238 assert(pb_cache_bucket < ARRAY_SIZE(ws->bo_cache.buckets));
1239
1240 /* Get a buffer from the cache. */
1241 bo = (struct amdgpu_winsys_bo*)
1242 pb_cache_reclaim_buffer(&ws->bo_cache, size, alignment, usage,
1243 pb_cache_bucket);
1244 if (bo)
1245 return &bo->base;
1246
1247 /* Create a new one. */
1248 bo = amdgpu_create_bo(ws, size, alignment, usage, domain, flags,
1249 pb_cache_bucket);
1250 if (!bo) {
1251 /* Clear the cache and try again. */
1252 pb_slabs_reclaim(&ws->bo_slabs);
1253 pb_cache_release_all_buffers(&ws->bo_cache);
1254 bo = amdgpu_create_bo(ws, size, alignment, usage, domain, flags,
1255 pb_cache_bucket);
1256 if (!bo)
1257 return NULL;
1258 }
1259
1260 bo->u.real.use_reusable_pool = true;
1261 return &bo->base;
1262 }
1263
1264 static struct pb_buffer *amdgpu_bo_from_handle(struct radeon_winsys *rws,
1265 struct winsys_handle *whandle,
1266 unsigned *stride,
1267 unsigned *offset)
1268 {
1269 struct amdgpu_winsys *ws = amdgpu_winsys(rws);
1270 struct amdgpu_winsys_bo *bo;
1271 enum amdgpu_bo_handle_type type;
1272 struct amdgpu_bo_import_result result = {0};
1273 uint64_t va;
1274 amdgpu_va_handle va_handle;
1275 struct amdgpu_bo_info info = {0};
1276 enum radeon_bo_domain initial = 0;
1277 int r;
1278
1279 /* Initialize the structure. */
1280 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
1281 if (!bo) {
1282 return NULL;
1283 }
1284
1285 switch (whandle->type) {
1286 case DRM_API_HANDLE_TYPE_SHARED:
1287 type = amdgpu_bo_handle_type_gem_flink_name;
1288 break;
1289 case DRM_API_HANDLE_TYPE_FD:
1290 type = amdgpu_bo_handle_type_dma_buf_fd;
1291 break;
1292 default:
1293 return NULL;
1294 }
1295
1296 r = amdgpu_bo_import(ws->dev, type, whandle->handle, &result);
1297 if (r)
1298 goto error;
1299
1300 /* Get initial domains. */
1301 r = amdgpu_bo_query_info(result.buf_handle, &info);
1302 if (r)
1303 goto error_query;
1304
1305 r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
1306 result.alloc_size, 1 << 20, 0, &va, &va_handle, 0);
1307 if (r)
1308 goto error_query;
1309
1310 r = amdgpu_bo_va_op(result.buf_handle, 0, result.alloc_size, va, 0, AMDGPU_VA_OP_MAP);
1311 if (r)
1312 goto error_va_map;
1313
1314 if (info.preferred_heap & AMDGPU_GEM_DOMAIN_VRAM)
1315 initial |= RADEON_DOMAIN_VRAM;
1316 if (info.preferred_heap & AMDGPU_GEM_DOMAIN_GTT)
1317 initial |= RADEON_DOMAIN_GTT;
1318
1319
1320 pipe_reference_init(&bo->base.reference, 1);
1321 bo->base.alignment = info.phys_alignment;
1322 bo->bo = result.buf_handle;
1323 bo->base.size = result.alloc_size;
1324 bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
1325 bo->ws = ws;
1326 bo->va = va;
1327 bo->u.real.va_handle = va_handle;
1328 bo->initial_domain = initial;
1329 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
1330 bo->is_shared = true;
1331
1332 if (stride)
1333 *stride = whandle->stride;
1334 if (offset)
1335 *offset = whandle->offset;
1336
1337 if (bo->initial_domain & RADEON_DOMAIN_VRAM)
1338 ws->allocated_vram += align64(bo->base.size, ws->info.gart_page_size);
1339 else if (bo->initial_domain & RADEON_DOMAIN_GTT)
1340 ws->allocated_gtt += align64(bo->base.size, ws->info.gart_page_size);
1341
1342 amdgpu_add_buffer_to_global_list(bo);
1343
1344 return &bo->base;
1345
1346 error_va_map:
1347 amdgpu_va_range_free(va_handle);
1348
1349 error_query:
1350 amdgpu_bo_free(result.buf_handle);
1351
1352 error:
1353 FREE(bo);
1354 return NULL;
1355 }
1356
1357 static bool amdgpu_bo_get_handle(struct pb_buffer *buffer,
1358 unsigned stride, unsigned offset,
1359 unsigned slice_size,
1360 struct winsys_handle *whandle)
1361 {
1362 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(buffer);
1363 enum amdgpu_bo_handle_type type;
1364 int r;
1365
1366 if (!bo->bo) {
1367 offset += bo->va - bo->u.slab.real->va;
1368 bo = bo->u.slab.real;
1369 }
1370
1371 bo->u.real.use_reusable_pool = false;
1372
1373 switch (whandle->type) {
1374 case DRM_API_HANDLE_TYPE_SHARED:
1375 type = amdgpu_bo_handle_type_gem_flink_name;
1376 break;
1377 case DRM_API_HANDLE_TYPE_FD:
1378 type = amdgpu_bo_handle_type_dma_buf_fd;
1379 break;
1380 case DRM_API_HANDLE_TYPE_KMS:
1381 type = amdgpu_bo_handle_type_kms;
1382 break;
1383 default:
1384 return false;
1385 }
1386
1387 r = amdgpu_bo_export(bo->bo, type, &whandle->handle);
1388 if (r)
1389 return false;
1390
1391 whandle->stride = stride;
1392 whandle->offset = offset;
1393 whandle->offset += slice_size * whandle->layer;
1394 bo->is_shared = true;
1395 return true;
1396 }
1397
1398 static struct pb_buffer *amdgpu_bo_from_ptr(struct radeon_winsys *rws,
1399 void *pointer, uint64_t size)
1400 {
1401 struct amdgpu_winsys *ws = amdgpu_winsys(rws);
1402 amdgpu_bo_handle buf_handle;
1403 struct amdgpu_winsys_bo *bo;
1404 uint64_t va;
1405 amdgpu_va_handle va_handle;
1406
1407 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
1408 if (!bo)
1409 return NULL;
1410
1411 if (amdgpu_create_bo_from_user_mem(ws->dev, pointer, size, &buf_handle))
1412 goto error;
1413
1414 if (amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
1415 size, 1 << 12, 0, &va, &va_handle, 0))
1416 goto error_va_alloc;
1417
1418 if (amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP))
1419 goto error_va_map;
1420
1421 /* Initialize it. */
1422 pipe_reference_init(&bo->base.reference, 1);
1423 bo->bo = buf_handle;
1424 bo->base.alignment = 0;
1425 bo->base.size = size;
1426 bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
1427 bo->ws = ws;
1428 bo->user_ptr = pointer;
1429 bo->va = va;
1430 bo->u.real.va_handle = va_handle;
1431 bo->initial_domain = RADEON_DOMAIN_GTT;
1432 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
1433
1434 ws->allocated_gtt += align64(bo->base.size, ws->info.gart_page_size);
1435
1436 amdgpu_add_buffer_to_global_list(bo);
1437
1438 return (struct pb_buffer*)bo;
1439
1440 error_va_map:
1441 amdgpu_va_range_free(va_handle);
1442
1443 error_va_alloc:
1444 amdgpu_bo_free(buf_handle);
1445
1446 error:
1447 FREE(bo);
1448 return NULL;
1449 }
1450
1451 static bool amdgpu_bo_is_user_ptr(struct pb_buffer *buf)
1452 {
1453 return ((struct amdgpu_winsys_bo*)buf)->user_ptr != NULL;
1454 }
1455
1456 static uint64_t amdgpu_bo_get_va(struct pb_buffer *buf)
1457 {
1458 return ((struct amdgpu_winsys_bo*)buf)->va;
1459 }
1460
1461 void amdgpu_bo_init_functions(struct amdgpu_winsys *ws)
1462 {
1463 ws->base.buffer_set_metadata = amdgpu_buffer_set_metadata;
1464 ws->base.buffer_get_metadata = amdgpu_buffer_get_metadata;
1465 ws->base.buffer_map = amdgpu_bo_map;
1466 ws->base.buffer_unmap = amdgpu_bo_unmap;
1467 ws->base.buffer_wait = amdgpu_bo_wait;
1468 ws->base.buffer_create = amdgpu_bo_create;
1469 ws->base.buffer_from_handle = amdgpu_bo_from_handle;
1470 ws->base.buffer_from_ptr = amdgpu_bo_from_ptr;
1471 ws->base.buffer_is_user_ptr = amdgpu_bo_is_user_ptr;
1472 ws->base.buffer_get_handle = amdgpu_bo_get_handle;
1473 ws->base.buffer_commit = amdgpu_bo_sparse_commit;
1474 ws->base.buffer_get_virtual_address = amdgpu_bo_get_va;
1475 ws->base.buffer_get_initial_domain = amdgpu_bo_get_initial_domain;
1476 }