gallium/radeon: rename winsys buffer_get/set_tiling to buffer_get/set_metadata
[mesa.git] / src / gallium / winsys / amdgpu / drm / amdgpu_bo.c
1 /*
2 * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
3 * Copyright © 2015 Advanced Micro Devices, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
16 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
18 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * The above copyright notice and this permission notice (including the
24 * next paragraph) shall be included in all copies or substantial portions
25 * of the Software.
26 */
27 /*
28 * Authors:
29 * Marek Olšák <maraeo@gmail.com>
30 */
31
32 #include "amdgpu_cs.h"
33
34 #include "os/os_time.h"
35 #include "state_tracker/drm_driver.h"
36 #include <amdgpu_drm.h>
37 #include <xf86drm.h>
38 #include <stdio.h>
39
40 static inline struct amdgpu_winsys_bo *amdgpu_winsys_bo(struct pb_buffer *bo)
41 {
42 return (struct amdgpu_winsys_bo *)bo;
43 }
44
45 static bool amdgpu_bo_wait(struct pb_buffer *_buf, uint64_t timeout,
46 enum radeon_bo_usage usage)
47 {
48 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
49 struct amdgpu_winsys *ws = bo->ws;
50 int i;
51
52 if (bo->is_shared) {
53 /* We can't use user fences for shared buffers, because user fences
54 * are local to this process only. If we want to wait for all buffer
55 * uses in all processes, we have to use amdgpu_bo_wait_for_idle.
56 */
57 bool buffer_busy = true;
58 int r;
59
60 r = amdgpu_bo_wait_for_idle(bo->bo, timeout, &buffer_busy);
61 if (r)
62 fprintf(stderr, "%s: amdgpu_bo_wait_for_idle failed %i\n", __func__,
63 r);
64 return !buffer_busy;
65 }
66
67 if (timeout == 0) {
68 /* Timeout == 0 is quite simple. */
69 pipe_mutex_lock(ws->bo_fence_lock);
70 for (i = 0; i < RING_LAST; i++)
71 if (bo->fence[i]) {
72 if (amdgpu_fence_wait(bo->fence[i], 0, false)) {
73 /* Release the idle fence to avoid checking it again later. */
74 amdgpu_fence_reference(&bo->fence[i], NULL);
75 } else {
76 pipe_mutex_unlock(ws->bo_fence_lock);
77 return false;
78 }
79 }
80 pipe_mutex_unlock(ws->bo_fence_lock);
81 return true;
82
83 } else {
84 struct pipe_fence_handle *fence[RING_LAST] = {};
85 bool fence_idle[RING_LAST] = {};
86 bool buffer_idle = true;
87 int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
88
89 /* Take references to all fences, so that we can wait for them
90 * without the lock. */
91 pipe_mutex_lock(ws->bo_fence_lock);
92 for (i = 0; i < RING_LAST; i++)
93 amdgpu_fence_reference(&fence[i], bo->fence[i]);
94 pipe_mutex_unlock(ws->bo_fence_lock);
95
96 /* Now wait for the fences. */
97 for (i = 0; i < RING_LAST; i++) {
98 if (fence[i]) {
99 if (amdgpu_fence_wait(fence[i], abs_timeout, true))
100 fence_idle[i] = true;
101 else
102 buffer_idle = false;
103 }
104 }
105
106 /* Release idle fences to avoid checking them again later. */
107 pipe_mutex_lock(ws->bo_fence_lock);
108 for (i = 0; i < RING_LAST; i++) {
109 if (fence[i] == bo->fence[i] && fence_idle[i])
110 amdgpu_fence_reference(&bo->fence[i], NULL);
111
112 amdgpu_fence_reference(&fence[i], NULL);
113 }
114 pipe_mutex_unlock(ws->bo_fence_lock);
115
116 return buffer_idle;
117 }
118 }
119
120 static enum radeon_bo_domain amdgpu_bo_get_initial_domain(
121 struct pb_buffer *buf)
122 {
123 return ((struct amdgpu_winsys_bo*)buf)->initial_domain;
124 }
125
126 void amdgpu_bo_destroy(struct pb_buffer *_buf)
127 {
128 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
129 int i;
130
131 pipe_mutex_lock(bo->ws->global_bo_list_lock);
132 LIST_DEL(&bo->global_list_item);
133 bo->ws->num_buffers--;
134 pipe_mutex_unlock(bo->ws->global_bo_list_lock);
135
136 amdgpu_bo_va_op(bo->bo, 0, bo->base.size, bo->va, 0, AMDGPU_VA_OP_UNMAP);
137 amdgpu_va_range_free(bo->va_handle);
138 amdgpu_bo_free(bo->bo);
139
140 for (i = 0; i < RING_LAST; i++)
141 amdgpu_fence_reference(&bo->fence[i], NULL);
142
143 if (bo->initial_domain & RADEON_DOMAIN_VRAM)
144 bo->ws->allocated_vram -= align(bo->base.size, bo->ws->gart_page_size);
145 else if (bo->initial_domain & RADEON_DOMAIN_GTT)
146 bo->ws->allocated_gtt -= align(bo->base.size, bo->ws->gart_page_size);
147 FREE(bo);
148 }
149
150 static void amdgpu_bo_destroy_or_cache(struct pb_buffer *_buf)
151 {
152 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
153
154 if (bo->use_reusable_pool)
155 pb_cache_add_buffer(&bo->cache_entry);
156 else
157 amdgpu_bo_destroy(_buf);
158 }
159
160 static void *amdgpu_bo_map(struct pb_buffer *buf,
161 struct radeon_winsys_cs *rcs,
162 enum pipe_transfer_usage usage)
163 {
164 struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
165 struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
166 int r;
167 void *cpu = NULL;
168
169 /* If it's not unsynchronized bo_map, flush CS if needed and then wait. */
170 if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
171 /* DONTBLOCK doesn't make sense with UNSYNCHRONIZED. */
172 if (usage & PIPE_TRANSFER_DONTBLOCK) {
173 if (!(usage & PIPE_TRANSFER_WRITE)) {
174 /* Mapping for read.
175 *
176 * Since we are mapping for read, we don't need to wait
177 * if the GPU is using the buffer for read too
178 * (neither one is changing it).
179 *
180 * Only check whether the buffer is being used for write. */
181 if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
182 RADEON_USAGE_WRITE)) {
183 cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
184 return NULL;
185 }
186
187 if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
188 RADEON_USAGE_WRITE)) {
189 return NULL;
190 }
191 } else {
192 if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo)) {
193 cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC, NULL);
194 return NULL;
195 }
196
197 if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
198 RADEON_USAGE_READWRITE)) {
199 return NULL;
200 }
201 }
202 } else {
203 uint64_t time = os_time_get_nano();
204
205 if (!(usage & PIPE_TRANSFER_WRITE)) {
206 /* Mapping for read.
207 *
208 * Since we are mapping for read, we don't need to wait
209 * if the GPU is using the buffer for read too
210 * (neither one is changing it).
211 *
212 * Only check whether the buffer is being used for write. */
213 if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
214 RADEON_USAGE_WRITE)) {
215 cs->flush_cs(cs->flush_data, 0, NULL);
216 }
217 amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
218 RADEON_USAGE_WRITE);
219 } else {
220 /* Mapping for write. */
221 if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo))
222 cs->flush_cs(cs->flush_data, 0, NULL);
223
224 amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
225 RADEON_USAGE_READWRITE);
226 }
227
228 bo->ws->buffer_wait_time += os_time_get_nano() - time;
229 }
230 }
231
232 /* If the buffer is created from user memory, return the user pointer. */
233 if (bo->user_ptr)
234 return bo->user_ptr;
235
236 r = amdgpu_bo_cpu_map(bo->bo, &cpu);
237 if (r) {
238 /* Clear the cache and try again. */
239 pb_cache_release_all_buffers(&bo->ws->bo_cache);
240 r = amdgpu_bo_cpu_map(bo->bo, &cpu);
241 }
242 return r ? NULL : cpu;
243 }
244
245 static void amdgpu_bo_unmap(struct pb_buffer *buf)
246 {
247 struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
248
249 amdgpu_bo_cpu_unmap(bo->bo);
250 }
251
252 static const struct pb_vtbl amdgpu_winsys_bo_vtbl = {
253 amdgpu_bo_destroy_or_cache
254 /* other functions are never called */
255 };
256
257 static void amdgpu_add_buffer_to_global_list(struct amdgpu_winsys_bo *bo)
258 {
259 struct amdgpu_winsys *ws = bo->ws;
260
261 pipe_mutex_lock(ws->global_bo_list_lock);
262 LIST_ADDTAIL(&bo->global_list_item, &ws->global_bo_list);
263 ws->num_buffers++;
264 pipe_mutex_unlock(ws->global_bo_list_lock);
265 }
266
267 static struct amdgpu_winsys_bo *amdgpu_create_bo(struct amdgpu_winsys *ws,
268 unsigned size,
269 unsigned alignment,
270 unsigned usage,
271 enum radeon_bo_domain initial_domain,
272 unsigned flags)
273 {
274 struct amdgpu_bo_alloc_request request = {0};
275 amdgpu_bo_handle buf_handle;
276 uint64_t va = 0;
277 struct amdgpu_winsys_bo *bo;
278 amdgpu_va_handle va_handle;
279 int r;
280
281 assert(initial_domain & RADEON_DOMAIN_VRAM_GTT);
282 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
283 if (!bo) {
284 return NULL;
285 }
286
287 pb_cache_init_entry(&ws->bo_cache, &bo->cache_entry, &bo->base);
288 request.alloc_size = size;
289 request.phys_alignment = alignment;
290
291 if (initial_domain & RADEON_DOMAIN_VRAM)
292 request.preferred_heap |= AMDGPU_GEM_DOMAIN_VRAM;
293 if (initial_domain & RADEON_DOMAIN_GTT)
294 request.preferred_heap |= AMDGPU_GEM_DOMAIN_GTT;
295
296 if (flags & RADEON_FLAG_CPU_ACCESS)
297 request.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
298 if (flags & RADEON_FLAG_NO_CPU_ACCESS)
299 request.flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
300 if (flags & RADEON_FLAG_GTT_WC)
301 request.flags |= AMDGPU_GEM_CREATE_CPU_GTT_USWC;
302
303 r = amdgpu_bo_alloc(ws->dev, &request, &buf_handle);
304 if (r) {
305 fprintf(stderr, "amdgpu: Failed to allocate a buffer:\n");
306 fprintf(stderr, "amdgpu: size : %d bytes\n", size);
307 fprintf(stderr, "amdgpu: alignment : %d bytes\n", alignment);
308 fprintf(stderr, "amdgpu: domains : %d\n", initial_domain);
309 goto error_bo_alloc;
310 }
311
312 r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
313 size, alignment, 0, &va, &va_handle, 0);
314 if (r)
315 goto error_va_alloc;
316
317 r = amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP);
318 if (r)
319 goto error_va_map;
320
321 pipe_reference_init(&bo->base.reference, 1);
322 bo->base.alignment = alignment;
323 bo->base.usage = usage;
324 bo->base.size = size;
325 bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
326 bo->ws = ws;
327 bo->bo = buf_handle;
328 bo->va = va;
329 bo->va_handle = va_handle;
330 bo->initial_domain = initial_domain;
331 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
332
333 if (initial_domain & RADEON_DOMAIN_VRAM)
334 ws->allocated_vram += align(size, ws->gart_page_size);
335 else if (initial_domain & RADEON_DOMAIN_GTT)
336 ws->allocated_gtt += align(size, ws->gart_page_size);
337
338 amdgpu_add_buffer_to_global_list(bo);
339
340 return bo;
341
342 error_va_map:
343 amdgpu_va_range_free(va_handle);
344
345 error_va_alloc:
346 amdgpu_bo_free(buf_handle);
347
348 error_bo_alloc:
349 FREE(bo);
350 return NULL;
351 }
352
353 bool amdgpu_bo_can_reclaim(struct pb_buffer *_buf)
354 {
355 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
356
357 if (amdgpu_bo_is_referenced_by_any_cs(bo)) {
358 return false;
359 }
360
361 return amdgpu_bo_wait(_buf, 0, RADEON_USAGE_READWRITE);
362 }
363
364 static unsigned eg_tile_split(unsigned tile_split)
365 {
366 switch (tile_split) {
367 case 0: tile_split = 64; break;
368 case 1: tile_split = 128; break;
369 case 2: tile_split = 256; break;
370 case 3: tile_split = 512; break;
371 default:
372 case 4: tile_split = 1024; break;
373 case 5: tile_split = 2048; break;
374 case 6: tile_split = 4096; break;
375 }
376 return tile_split;
377 }
378
379 static unsigned eg_tile_split_rev(unsigned eg_tile_split)
380 {
381 switch (eg_tile_split) {
382 case 64: return 0;
383 case 128: return 1;
384 case 256: return 2;
385 case 512: return 3;
386 default:
387 case 1024: return 4;
388 case 2048: return 5;
389 case 4096: return 6;
390 }
391 }
392
393 static void amdgpu_buffer_get_metadata(struct pb_buffer *_buf,
394 struct radeon_bo_metadata *md)
395 {
396 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
397 struct amdgpu_bo_info info = {0};
398 uint32_t tiling_flags;
399 int r;
400
401 r = amdgpu_bo_query_info(bo->bo, &info);
402 if (r)
403 return;
404
405 tiling_flags = info.metadata.tiling_info;
406
407 md->microtile = RADEON_LAYOUT_LINEAR;
408 md->macrotile = RADEON_LAYOUT_LINEAR;
409
410 if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 4) /* 2D_TILED_THIN1 */
411 md->macrotile = RADEON_LAYOUT_TILED;
412 else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 2) /* 1D_TILED_THIN1 */
413 md->microtile = RADEON_LAYOUT_TILED;
414
415 md->bankw = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_WIDTH);
416 md->bankh = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_HEIGHT);
417 md->tile_split = eg_tile_split(AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT));
418 md->mtilea = 1 << AMDGPU_TILING_GET(tiling_flags, MACRO_TILE_ASPECT);
419 md->scanout = AMDGPU_TILING_GET(tiling_flags, MICRO_TILE_MODE) == 0; /* DISPLAY */
420 }
421
422 static void amdgpu_buffer_set_metadata(struct pb_buffer *_buf,
423 struct radeon_bo_metadata *md)
424 {
425 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
426 struct amdgpu_bo_metadata metadata = {0};
427 uint32_t tiling_flags = 0;
428
429 if (md->macrotile == RADEON_LAYOUT_TILED)
430 tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 4); /* 2D_TILED_THIN1 */
431 else if (md->microtile == RADEON_LAYOUT_TILED)
432 tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 2); /* 1D_TILED_THIN1 */
433 else
434 tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 1); /* LINEAR_ALIGNED */
435
436 tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, md->pipe_config);
437 tiling_flags |= AMDGPU_TILING_SET(BANK_WIDTH, util_logbase2(md->bankw));
438 tiling_flags |= AMDGPU_TILING_SET(BANK_HEIGHT, util_logbase2(md->bankh));
439 if (md->tile_split)
440 tiling_flags |= AMDGPU_TILING_SET(TILE_SPLIT, eg_tile_split_rev(md->tile_split));
441 tiling_flags |= AMDGPU_TILING_SET(MACRO_TILE_ASPECT, util_logbase2(md->mtilea));
442 tiling_flags |= AMDGPU_TILING_SET(NUM_BANKS, util_logbase2(md->num_banks)-1);
443
444 if (md->scanout)
445 tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 0); /* DISPLAY_MICRO_TILING */
446 else
447 tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 1); /* THIN_MICRO_TILING */
448
449 metadata.tiling_info = tiling_flags;
450
451 amdgpu_bo_set_metadata(bo->bo, &metadata);
452 }
453
454 static struct pb_buffer *
455 amdgpu_bo_create(struct radeon_winsys *rws,
456 unsigned size,
457 unsigned alignment,
458 boolean use_reusable_pool,
459 enum radeon_bo_domain domain,
460 enum radeon_bo_flag flags)
461 {
462 struct amdgpu_winsys *ws = amdgpu_winsys(rws);
463 struct amdgpu_winsys_bo *bo;
464 unsigned usage = 0;
465
466 /* Don't use VRAM if the GPU doesn't have much. This is only the initial
467 * domain. The kernel is free to move the buffer if it wants to.
468 *
469 * 64MB means no VRAM by todays standards.
470 */
471 if (domain & RADEON_DOMAIN_VRAM && ws->info.vram_size <= 64*1024*1024) {
472 domain = RADEON_DOMAIN_GTT;
473 flags = RADEON_FLAG_GTT_WC;
474 }
475
476 /* Align size to page size. This is the minimum alignment for normal
477 * BOs. Aligning this here helps the cached bufmgr. Especially small BOs,
478 * like constant/uniform buffers, can benefit from better and more reuse.
479 */
480 size = align(size, ws->gart_page_size);
481
482 /* Only set one usage bit each for domains and flags, or the cache manager
483 * might consider different sets of domains / flags compatible
484 */
485 if (domain == RADEON_DOMAIN_VRAM_GTT)
486 usage = 1 << 2;
487 else
488 usage = domain >> 1;
489 assert(flags < sizeof(usage) * 8 - 3);
490 usage |= 1 << (flags + 3);
491
492 /* Get a buffer from the cache. */
493 if (use_reusable_pool) {
494 bo = (struct amdgpu_winsys_bo*)
495 pb_cache_reclaim_buffer(&ws->bo_cache, size, alignment,
496 usage);
497 if (bo)
498 return &bo->base;
499 }
500
501 /* Create a new one. */
502 bo = amdgpu_create_bo(ws, size, alignment, usage, domain, flags);
503 if (!bo) {
504 /* Clear the cache and try again. */
505 pb_cache_release_all_buffers(&ws->bo_cache);
506 bo = amdgpu_create_bo(ws, size, alignment, usage, domain, flags);
507 if (!bo)
508 return NULL;
509 }
510
511 bo->use_reusable_pool = use_reusable_pool;
512 return &bo->base;
513 }
514
515 static struct pb_buffer *amdgpu_bo_from_handle(struct radeon_winsys *rws,
516 struct winsys_handle *whandle,
517 unsigned *stride)
518 {
519 struct amdgpu_winsys *ws = amdgpu_winsys(rws);
520 struct amdgpu_winsys_bo *bo;
521 enum amdgpu_bo_handle_type type;
522 struct amdgpu_bo_import_result result = {0};
523 uint64_t va;
524 amdgpu_va_handle va_handle;
525 struct amdgpu_bo_info info = {0};
526 enum radeon_bo_domain initial = 0;
527 int r;
528
529 /* Initialize the structure. */
530 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
531 if (!bo) {
532 return NULL;
533 }
534
535 switch (whandle->type) {
536 case DRM_API_HANDLE_TYPE_SHARED:
537 type = amdgpu_bo_handle_type_gem_flink_name;
538 break;
539 case DRM_API_HANDLE_TYPE_FD:
540 type = amdgpu_bo_handle_type_dma_buf_fd;
541 break;
542 default:
543 return NULL;
544 }
545
546 r = amdgpu_bo_import(ws->dev, type, whandle->handle, &result);
547 if (r)
548 goto error;
549
550 /* Get initial domains. */
551 r = amdgpu_bo_query_info(result.buf_handle, &info);
552 if (r)
553 goto error_query;
554
555 r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
556 result.alloc_size, 1 << 20, 0, &va, &va_handle, 0);
557 if (r)
558 goto error_query;
559
560 r = amdgpu_bo_va_op(result.buf_handle, 0, result.alloc_size, va, 0, AMDGPU_VA_OP_MAP);
561 if (r)
562 goto error_va_map;
563
564 if (info.preferred_heap & AMDGPU_GEM_DOMAIN_VRAM)
565 initial |= RADEON_DOMAIN_VRAM;
566 if (info.preferred_heap & AMDGPU_GEM_DOMAIN_GTT)
567 initial |= RADEON_DOMAIN_GTT;
568
569
570 pipe_reference_init(&bo->base.reference, 1);
571 bo->base.alignment = info.phys_alignment;
572 bo->base.usage = PB_USAGE_GPU_WRITE | PB_USAGE_GPU_READ;
573 bo->bo = result.buf_handle;
574 bo->base.size = result.alloc_size;
575 bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
576 bo->ws = ws;
577 bo->va = va;
578 bo->va_handle = va_handle;
579 bo->initial_domain = initial;
580 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
581 bo->is_shared = true;
582
583 if (stride)
584 *stride = whandle->stride;
585
586 if (bo->initial_domain & RADEON_DOMAIN_VRAM)
587 ws->allocated_vram += align(bo->base.size, ws->gart_page_size);
588 else if (bo->initial_domain & RADEON_DOMAIN_GTT)
589 ws->allocated_gtt += align(bo->base.size, ws->gart_page_size);
590
591 amdgpu_add_buffer_to_global_list(bo);
592
593 return &bo->base;
594
595 error_va_map:
596 amdgpu_va_range_free(va_handle);
597
598 error_query:
599 amdgpu_bo_free(result.buf_handle);
600
601 error:
602 FREE(bo);
603 return NULL;
604 }
605
606 static boolean amdgpu_bo_get_handle(struct pb_buffer *buffer,
607 unsigned stride,
608 struct winsys_handle *whandle)
609 {
610 struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(buffer);
611 enum amdgpu_bo_handle_type type;
612 int r;
613
614 bo->use_reusable_pool = false;
615
616 switch (whandle->type) {
617 case DRM_API_HANDLE_TYPE_SHARED:
618 type = amdgpu_bo_handle_type_gem_flink_name;
619 break;
620 case DRM_API_HANDLE_TYPE_FD:
621 type = amdgpu_bo_handle_type_dma_buf_fd;
622 break;
623 case DRM_API_HANDLE_TYPE_KMS:
624 type = amdgpu_bo_handle_type_kms;
625 break;
626 default:
627 return FALSE;
628 }
629
630 r = amdgpu_bo_export(bo->bo, type, &whandle->handle);
631 if (r)
632 return FALSE;
633
634 whandle->stride = stride;
635 bo->is_shared = true;
636 return TRUE;
637 }
638
639 static struct pb_buffer *amdgpu_bo_from_ptr(struct radeon_winsys *rws,
640 void *pointer, unsigned size)
641 {
642 struct amdgpu_winsys *ws = amdgpu_winsys(rws);
643 amdgpu_bo_handle buf_handle;
644 struct amdgpu_winsys_bo *bo;
645 uint64_t va;
646 amdgpu_va_handle va_handle;
647
648 bo = CALLOC_STRUCT(amdgpu_winsys_bo);
649 if (!bo)
650 return NULL;
651
652 if (amdgpu_create_bo_from_user_mem(ws->dev, pointer, size, &buf_handle))
653 goto error;
654
655 if (amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
656 size, 1 << 12, 0, &va, &va_handle, 0))
657 goto error_va_alloc;
658
659 if (amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP))
660 goto error_va_map;
661
662 /* Initialize it. */
663 pipe_reference_init(&bo->base.reference, 1);
664 bo->bo = buf_handle;
665 bo->base.alignment = 0;
666 bo->base.usage = PB_USAGE_GPU_WRITE | PB_USAGE_GPU_READ;
667 bo->base.size = size;
668 bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
669 bo->ws = ws;
670 bo->user_ptr = pointer;
671 bo->va = va;
672 bo->va_handle = va_handle;
673 bo->initial_domain = RADEON_DOMAIN_GTT;
674 bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
675
676 ws->allocated_gtt += align(bo->base.size, ws->gart_page_size);
677
678 amdgpu_add_buffer_to_global_list(bo);
679
680 return (struct pb_buffer*)bo;
681
682 error_va_map:
683 amdgpu_va_range_free(va_handle);
684
685 error_va_alloc:
686 amdgpu_bo_free(buf_handle);
687
688 error:
689 FREE(bo);
690 return NULL;
691 }
692
693 static bool amdgpu_bo_is_user_ptr(struct pb_buffer *buf)
694 {
695 return ((struct amdgpu_winsys_bo*)buf)->user_ptr != NULL;
696 }
697
698 static uint64_t amdgpu_bo_get_va(struct pb_buffer *buf)
699 {
700 return ((struct amdgpu_winsys_bo*)buf)->va;
701 }
702
703 void amdgpu_bo_init_functions(struct amdgpu_winsys *ws)
704 {
705 ws->base.buffer_set_metadata = amdgpu_buffer_set_metadata;
706 ws->base.buffer_get_metadata = amdgpu_buffer_get_metadata;
707 ws->base.buffer_map = amdgpu_bo_map;
708 ws->base.buffer_unmap = amdgpu_bo_unmap;
709 ws->base.buffer_wait = amdgpu_bo_wait;
710 ws->base.buffer_create = amdgpu_bo_create;
711 ws->base.buffer_from_handle = amdgpu_bo_from_handle;
712 ws->base.buffer_from_ptr = amdgpu_bo_from_ptr;
713 ws->base.buffer_is_user_ptr = amdgpu_bo_is_user_ptr;
714 ws->base.buffer_get_handle = amdgpu_bo_get_handle;
715 ws->base.buffer_get_virtual_address = amdgpu_bo_get_va;
716 ws->base.buffer_get_initial_domain = amdgpu_bo_get_initial_domain;
717 }