2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 #include "drm-uapi/amdgpu_drm.h"
32 #include "util/u_memory.h"
34 #include "radv_radeon_winsys.h"
35 #include "radv_amdgpu_cs.h"
36 #include "radv_amdgpu_bo.h"
41 VIRTUAL_BUFFER_HASH_TABLE_SIZE
= 1024
44 struct radv_amdgpu_cs
{
45 struct radeon_cmdbuf base
;
46 struct radv_amdgpu_winsys
*ws
;
48 struct amdgpu_cs_ib_info ib
;
50 struct radeon_winsys_bo
*ib_buffer
;
52 unsigned max_num_buffers
;
54 struct drm_amdgpu_bo_list_entry
*handles
;
56 struct radeon_winsys_bo
**old_ib_buffers
;
57 unsigned num_old_ib_buffers
;
58 unsigned max_num_old_ib_buffers
;
59 unsigned *ib_size_ptr
;
63 int buffer_hash_table
[1024];
66 unsigned num_virtual_buffers
;
67 unsigned max_num_virtual_buffers
;
68 struct radeon_winsys_bo
**virtual_buffers
;
69 int *virtual_buffer_hash_table
;
71 /* For chips that don't support chaining. */
72 struct radeon_cmdbuf
*old_cs_buffers
;
73 unsigned num_old_cs_buffers
;
76 static inline struct radv_amdgpu_cs
*
77 radv_amdgpu_cs(struct radeon_cmdbuf
*base
)
79 return (struct radv_amdgpu_cs
*)base
;
82 static int ring_to_hw_ip(enum ring_type ring
)
86 return AMDGPU_HW_IP_GFX
;
88 return AMDGPU_HW_IP_DMA
;
90 return AMDGPU_HW_IP_COMPUTE
;
92 unreachable("unsupported ring");
96 struct radv_amdgpu_cs_request
{
97 /** Specify flags with additional information */
100 /** Specify HW IP block type to which to send the IB. */
103 /** IP instance index if there are several IPs of the same type. */
104 unsigned ip_instance
;
107 * Specify ring index of the IP. We could have several rings
108 * in the same IP. E.g. 0 for SDMA0 and 1 for SDMA1.
113 * BO list handles used by this request.
115 struct drm_amdgpu_bo_list_entry
*handles
;
116 uint32_t num_handles
;
119 * Number of dependencies this Command submission needs to
120 * wait for before starting execution.
122 uint32_t number_of_dependencies
;
125 * Array of dependencies which need to be met before
126 * execution can start.
128 struct amdgpu_cs_fence
*dependencies
;
130 /** Number of IBs to submit in the field ibs. */
131 uint32_t number_of_ibs
;
134 * IBs to submit. Those IBs will be submit together as single entity
136 struct amdgpu_cs_ib_info
*ibs
;
139 * The returned sequence number for the command submission
144 * The fence information
146 struct amdgpu_cs_fence_info fence_info
;
150 static int radv_amdgpu_signal_sems(struct radv_amdgpu_ctx
*ctx
,
153 struct radv_winsys_sem_info
*sem_info
);
154 static int radv_amdgpu_cs_submit(struct radv_amdgpu_ctx
*ctx
,
155 struct radv_amdgpu_cs_request
*request
,
156 struct radv_winsys_sem_info
*sem_info
);
158 static void radv_amdgpu_request_to_fence(struct radv_amdgpu_ctx
*ctx
,
159 struct radv_amdgpu_fence
*fence
,
160 struct radv_amdgpu_cs_request
*req
)
162 fence
->fence
.context
= ctx
->ctx
;
163 fence
->fence
.ip_type
= req
->ip_type
;
164 fence
->fence
.ip_instance
= req
->ip_instance
;
165 fence
->fence
.ring
= req
->ring
;
166 fence
->fence
.fence
= req
->seq_no
;
167 fence
->user_ptr
= (volatile uint64_t*)(ctx
->fence_map
+ req
->ip_type
* MAX_RINGS_PER_TYPE
+ req
->ring
);
170 static struct radeon_winsys_fence
*radv_amdgpu_create_fence()
172 struct radv_amdgpu_fence
*fence
= calloc(1, sizeof(struct radv_amdgpu_fence
));
176 fence
->fence
.fence
= UINT64_MAX
;
177 return (struct radeon_winsys_fence
*)fence
;
180 static void radv_amdgpu_destroy_fence(struct radeon_winsys_fence
*_fence
)
182 struct radv_amdgpu_fence
*fence
= (struct radv_amdgpu_fence
*)_fence
;
186 static void radv_amdgpu_reset_fence(struct radeon_winsys_fence
*_fence
)
188 struct radv_amdgpu_fence
*fence
= (struct radv_amdgpu_fence
*)_fence
;
189 fence
->fence
.fence
= UINT64_MAX
;
192 static void radv_amdgpu_signal_fence(struct radeon_winsys_fence
*_fence
)
194 struct radv_amdgpu_fence
*fence
= (struct radv_amdgpu_fence
*)_fence
;
195 fence
->fence
.fence
= 0;
198 static bool radv_amdgpu_is_fence_waitable(struct radeon_winsys_fence
*_fence
)
200 struct radv_amdgpu_fence
*fence
= (struct radv_amdgpu_fence
*)_fence
;
201 return fence
->fence
.fence
< UINT64_MAX
;
204 static bool radv_amdgpu_fence_wait(struct radeon_winsys
*_ws
,
205 struct radeon_winsys_fence
*_fence
,
209 struct radv_amdgpu_fence
*fence
= (struct radv_amdgpu_fence
*)_fence
;
210 unsigned flags
= absolute
? AMDGPU_QUERY_FENCE_TIMEOUT_IS_ABSOLUTE
: 0;
212 uint32_t expired
= 0;
214 /* Special casing 0 and UINT64_MAX so that they work without user_ptr/fence.ctx */
215 if (fence
->fence
.fence
== UINT64_MAX
)
218 if (fence
->fence
.fence
== 0)
221 if (fence
->user_ptr
) {
222 if (*fence
->user_ptr
>= fence
->fence
.fence
)
224 if (!absolute
&& !timeout
)
228 /* Now use the libdrm query. */
229 r
= amdgpu_cs_query_fence_status(&fence
->fence
,
235 fprintf(stderr
, "amdgpu: radv_amdgpu_cs_query_fence_status failed.\n");
246 static bool radv_amdgpu_fences_wait(struct radeon_winsys
*_ws
,
247 struct radeon_winsys_fence
*const *_fences
,
248 uint32_t fence_count
,
252 struct amdgpu_cs_fence
*fences
= malloc(sizeof(struct amdgpu_cs_fence
) * fence_count
);
254 uint32_t expired
= 0, first
= 0;
259 for (uint32_t i
= 0; i
< fence_count
; ++i
)
260 fences
[i
] = ((struct radv_amdgpu_fence
*)_fences
[i
])->fence
;
262 /* Now use the libdrm query. */
263 r
= amdgpu_cs_wait_fences(fences
, fence_count
, wait_all
,
264 timeout
, &expired
, &first
);
268 fprintf(stderr
, "amdgpu: amdgpu_cs_wait_fences failed.\n");
278 static void radv_amdgpu_cs_destroy(struct radeon_cmdbuf
*rcs
)
280 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(rcs
);
283 cs
->ws
->base
.buffer_destroy(cs
->ib_buffer
);
287 for (unsigned i
= 0; i
< cs
->num_old_ib_buffers
; ++i
)
288 cs
->ws
->base
.buffer_destroy(cs
->old_ib_buffers
[i
]);
290 for (unsigned i
= 0; i
< cs
->num_old_cs_buffers
; ++i
) {
291 struct radeon_cmdbuf
*rcs
= &cs
->old_cs_buffers
[i
];
295 free(cs
->old_cs_buffers
);
296 free(cs
->old_ib_buffers
);
297 free(cs
->virtual_buffers
);
298 free(cs
->virtual_buffer_hash_table
);
303 static void radv_amdgpu_init_cs(struct radv_amdgpu_cs
*cs
,
304 enum ring_type ring_type
)
306 for (int i
= 0; i
< ARRAY_SIZE(cs
->buffer_hash_table
); ++i
)
307 cs
->buffer_hash_table
[i
] = -1;
309 cs
->hw_ip
= ring_to_hw_ip(ring_type
);
312 static struct radeon_cmdbuf
*
313 radv_amdgpu_cs_create(struct radeon_winsys
*ws
,
314 enum ring_type ring_type
)
316 struct radv_amdgpu_cs
*cs
;
317 uint32_t ib_size
= 20 * 1024 * 4;
318 cs
= calloc(1, sizeof(struct radv_amdgpu_cs
));
322 cs
->ws
= radv_amdgpu_winsys(ws
);
323 radv_amdgpu_init_cs(cs
, ring_type
);
325 if (cs
->ws
->use_ib_bos
) {
326 cs
->ib_buffer
= ws
->buffer_create(ws
, ib_size
, 0,
328 RADEON_FLAG_CPU_ACCESS
|
329 RADEON_FLAG_NO_INTERPROCESS_SHARING
|
330 RADEON_FLAG_READ_ONLY
|
332 RADV_BO_PRIORITY_CS
);
333 if (!cs
->ib_buffer
) {
338 cs
->ib_mapped
= ws
->buffer_map(cs
->ib_buffer
);
339 if (!cs
->ib_mapped
) {
340 ws
->buffer_destroy(cs
->ib_buffer
);
345 cs
->ib
.ib_mc_address
= radv_amdgpu_winsys_bo(cs
->ib_buffer
)->base
.va
;
346 cs
->base
.buf
= (uint32_t *)cs
->ib_mapped
;
347 cs
->base
.max_dw
= ib_size
/ 4 - 4;
348 cs
->ib_size_ptr
= &cs
->ib
.size
;
351 ws
->cs_add_buffer(&cs
->base
, cs
->ib_buffer
);
353 uint32_t *buf
= malloc(16384);
359 cs
->base
.max_dw
= 4096;
365 static void radv_amdgpu_cs_grow(struct radeon_cmdbuf
*_cs
, size_t min_size
)
367 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(_cs
);
369 if (cs
->status
!= VK_SUCCESS
) {
374 if (!cs
->ws
->use_ib_bos
) {
375 const uint64_t limit_dws
= 0xffff8;
376 uint64_t ib_dws
= MAX2(cs
->base
.cdw
+ min_size
,
377 MIN2(cs
->base
.max_dw
* 2, limit_dws
));
379 /* The total ib size cannot exceed limit_dws dwords. */
380 if (ib_dws
> limit_dws
)
382 /* The maximum size in dwords has been reached,
383 * try to allocate a new one.
385 struct radeon_cmdbuf
*old_cs_buffers
=
386 realloc(cs
->old_cs_buffers
,
387 (cs
->num_old_cs_buffers
+ 1) * sizeof(*cs
->old_cs_buffers
));
388 if (!old_cs_buffers
) {
389 cs
->status
= VK_ERROR_OUT_OF_HOST_MEMORY
;
393 cs
->old_cs_buffers
= old_cs_buffers
;
395 /* Store the current one for submitting it later. */
396 cs
->old_cs_buffers
[cs
->num_old_cs_buffers
].cdw
= cs
->base
.cdw
;
397 cs
->old_cs_buffers
[cs
->num_old_cs_buffers
].max_dw
= cs
->base
.max_dw
;
398 cs
->old_cs_buffers
[cs
->num_old_cs_buffers
].buf
= cs
->base
.buf
;
399 cs
->num_old_cs_buffers
++;
401 /* Reset the cs, it will be re-allocated below. */
405 /* Re-compute the number of dwords to allocate. */
406 ib_dws
= MAX2(cs
->base
.cdw
+ min_size
,
407 MIN2(cs
->base
.max_dw
* 2, limit_dws
));
408 if (ib_dws
> limit_dws
) {
409 fprintf(stderr
, "amdgpu: Too high number of "
410 "dwords to allocate\n");
411 cs
->status
= VK_ERROR_OUT_OF_HOST_MEMORY
;
416 uint32_t *new_buf
= realloc(cs
->base
.buf
, ib_dws
* 4);
418 cs
->base
.buf
= new_buf
;
419 cs
->base
.max_dw
= ib_dws
;
421 cs
->status
= VK_ERROR_OUT_OF_HOST_MEMORY
;
427 uint64_t ib_size
= MAX2(min_size
* 4 + 16, cs
->base
.max_dw
* 4 * 2);
429 /* max that fits in the chain size field. */
430 ib_size
= MIN2(ib_size
, 0xfffff);
432 while (!cs
->base
.cdw
|| (cs
->base
.cdw
& 7) != 4)
433 radeon_emit(&cs
->base
, PKT3_NOP_PAD
);
435 *cs
->ib_size_ptr
|= cs
->base
.cdw
+ 4;
437 if (cs
->num_old_ib_buffers
== cs
->max_num_old_ib_buffers
) {
438 unsigned max_num_old_ib_buffers
=
439 MAX2(1, cs
->max_num_old_ib_buffers
* 2);
440 struct radeon_winsys_bo
**old_ib_buffers
=
441 realloc(cs
->old_ib_buffers
,
442 max_num_old_ib_buffers
* sizeof(void*));
443 if (!old_ib_buffers
) {
444 cs
->status
= VK_ERROR_OUT_OF_HOST_MEMORY
;
447 cs
->max_num_old_ib_buffers
= max_num_old_ib_buffers
;
448 cs
->old_ib_buffers
= old_ib_buffers
;
451 cs
->old_ib_buffers
[cs
->num_old_ib_buffers
++] = cs
->ib_buffer
;
453 cs
->ib_buffer
= cs
->ws
->base
.buffer_create(&cs
->ws
->base
, ib_size
, 0,
455 RADEON_FLAG_CPU_ACCESS
|
456 RADEON_FLAG_NO_INTERPROCESS_SHARING
|
457 RADEON_FLAG_READ_ONLY
|
459 RADV_BO_PRIORITY_CS
);
461 if (!cs
->ib_buffer
) {
463 cs
->status
= VK_ERROR_OUT_OF_DEVICE_MEMORY
;
464 cs
->ib_buffer
= cs
->old_ib_buffers
[--cs
->num_old_ib_buffers
];
467 cs
->ib_mapped
= cs
->ws
->base
.buffer_map(cs
->ib_buffer
);
468 if (!cs
->ib_mapped
) {
469 cs
->ws
->base
.buffer_destroy(cs
->ib_buffer
);
472 /* VK_ERROR_MEMORY_MAP_FAILED is not valid for vkEndCommandBuffer. */
473 cs
->status
= VK_ERROR_OUT_OF_DEVICE_MEMORY
;
474 cs
->ib_buffer
= cs
->old_ib_buffers
[--cs
->num_old_ib_buffers
];
477 cs
->ws
->base
.cs_add_buffer(&cs
->base
, cs
->ib_buffer
);
479 radeon_emit(&cs
->base
, PKT3(PKT3_INDIRECT_BUFFER_CIK
, 2, 0));
480 radeon_emit(&cs
->base
, radv_amdgpu_winsys_bo(cs
->ib_buffer
)->base
.va
);
481 radeon_emit(&cs
->base
, radv_amdgpu_winsys_bo(cs
->ib_buffer
)->base
.va
>> 32);
482 radeon_emit(&cs
->base
, S_3F2_CHAIN(1) | S_3F2_VALID(1));
484 cs
->ib_size_ptr
= cs
->base
.buf
+ cs
->base
.cdw
- 1;
486 cs
->base
.buf
= (uint32_t *)cs
->ib_mapped
;
488 cs
->base
.max_dw
= ib_size
/ 4 - 4;
492 static VkResult
radv_amdgpu_cs_finalize(struct radeon_cmdbuf
*_cs
)
494 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(_cs
);
496 if (cs
->ws
->use_ib_bos
) {
497 while (!cs
->base
.cdw
|| (cs
->base
.cdw
& 7) != 0)
498 radeon_emit(&cs
->base
, PKT3_NOP_PAD
);
500 *cs
->ib_size_ptr
|= cs
->base
.cdw
;
502 cs
->is_chained
= false;
508 static void radv_amdgpu_cs_reset(struct radeon_cmdbuf
*_cs
)
510 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(_cs
);
512 cs
->status
= VK_SUCCESS
;
514 for (unsigned i
= 0; i
< cs
->num_buffers
; ++i
) {
515 unsigned hash
= cs
->handles
[i
].bo_handle
&
516 (ARRAY_SIZE(cs
->buffer_hash_table
) - 1);
517 cs
->buffer_hash_table
[hash
] = -1;
520 for (unsigned i
= 0; i
< cs
->num_virtual_buffers
; ++i
) {
521 unsigned hash
= ((uintptr_t)cs
->virtual_buffers
[i
] >> 6) & (VIRTUAL_BUFFER_HASH_TABLE_SIZE
- 1);
522 cs
->virtual_buffer_hash_table
[hash
] = -1;
526 cs
->num_virtual_buffers
= 0;
528 if (cs
->ws
->use_ib_bos
) {
529 cs
->ws
->base
.cs_add_buffer(&cs
->base
, cs
->ib_buffer
);
531 for (unsigned i
= 0; i
< cs
->num_old_ib_buffers
; ++i
)
532 cs
->ws
->base
.buffer_destroy(cs
->old_ib_buffers
[i
]);
534 cs
->num_old_ib_buffers
= 0;
535 cs
->ib
.ib_mc_address
= radv_amdgpu_winsys_bo(cs
->ib_buffer
)->base
.va
;
536 cs
->ib_size_ptr
= &cs
->ib
.size
;
539 for (unsigned i
= 0; i
< cs
->num_old_cs_buffers
; ++i
) {
540 struct radeon_cmdbuf
*rcs
= &cs
->old_cs_buffers
[i
];
544 free(cs
->old_cs_buffers
);
545 cs
->old_cs_buffers
= NULL
;
546 cs
->num_old_cs_buffers
= 0;
550 static int radv_amdgpu_cs_find_buffer(struct radv_amdgpu_cs
*cs
,
553 unsigned hash
= bo
& (ARRAY_SIZE(cs
->buffer_hash_table
) - 1);
554 int index
= cs
->buffer_hash_table
[hash
];
559 if (cs
->handles
[index
].bo_handle
== bo
)
562 for (unsigned i
= 0; i
< cs
->num_buffers
; ++i
) {
563 if (cs
->handles
[i
].bo_handle
== bo
) {
564 cs
->buffer_hash_table
[hash
] = i
;
572 static void radv_amdgpu_cs_add_buffer_internal(struct radv_amdgpu_cs
*cs
,
573 uint32_t bo
, uint8_t priority
)
576 int index
= radv_amdgpu_cs_find_buffer(cs
, bo
);
581 if (cs
->num_buffers
== cs
->max_num_buffers
) {
582 unsigned new_count
= MAX2(1, cs
->max_num_buffers
* 2);
583 struct drm_amdgpu_bo_list_entry
*new_entries
=
584 realloc(cs
->handles
, new_count
* sizeof(struct drm_amdgpu_bo_list_entry
));
586 cs
->max_num_buffers
= new_count
;
587 cs
->handles
= new_entries
;
589 cs
->status
= VK_ERROR_OUT_OF_HOST_MEMORY
;
594 cs
->handles
[cs
->num_buffers
].bo_handle
= bo
;
595 cs
->handles
[cs
->num_buffers
].bo_priority
= priority
;
597 hash
= bo
& (ARRAY_SIZE(cs
->buffer_hash_table
) - 1);
598 cs
->buffer_hash_table
[hash
] = cs
->num_buffers
;
603 static void radv_amdgpu_cs_add_virtual_buffer(struct radeon_cmdbuf
*_cs
,
604 struct radeon_winsys_bo
*bo
)
606 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(_cs
);
607 unsigned hash
= ((uintptr_t)bo
>> 6) & (VIRTUAL_BUFFER_HASH_TABLE_SIZE
- 1);
610 if (!cs
->virtual_buffer_hash_table
) {
611 int *virtual_buffer_hash_table
=
612 malloc(VIRTUAL_BUFFER_HASH_TABLE_SIZE
* sizeof(int));
613 if (!virtual_buffer_hash_table
) {
614 cs
->status
= VK_ERROR_OUT_OF_HOST_MEMORY
;
617 cs
->virtual_buffer_hash_table
= virtual_buffer_hash_table
;
619 for (int i
= 0; i
< VIRTUAL_BUFFER_HASH_TABLE_SIZE
; ++i
)
620 cs
->virtual_buffer_hash_table
[i
] = -1;
623 if (cs
->virtual_buffer_hash_table
[hash
] >= 0) {
624 int idx
= cs
->virtual_buffer_hash_table
[hash
];
625 if (cs
->virtual_buffers
[idx
] == bo
) {
628 for (unsigned i
= 0; i
< cs
->num_virtual_buffers
; ++i
) {
629 if (cs
->virtual_buffers
[i
] == bo
) {
630 cs
->virtual_buffer_hash_table
[hash
] = i
;
636 if(cs
->max_num_virtual_buffers
<= cs
->num_virtual_buffers
) {
637 unsigned max_num_virtual_buffers
=
638 MAX2(2, cs
->max_num_virtual_buffers
* 2);
639 struct radeon_winsys_bo
**virtual_buffers
=
640 realloc(cs
->virtual_buffers
,
641 sizeof(struct radv_amdgpu_virtual_virtual_buffer
*) * max_num_virtual_buffers
);
642 if (!virtual_buffers
) {
643 cs
->status
= VK_ERROR_OUT_OF_HOST_MEMORY
;
646 cs
->max_num_virtual_buffers
= max_num_virtual_buffers
;
647 cs
->virtual_buffers
= virtual_buffers
;
650 cs
->virtual_buffers
[cs
->num_virtual_buffers
] = bo
;
652 cs
->virtual_buffer_hash_table
[hash
] = cs
->num_virtual_buffers
;
653 ++cs
->num_virtual_buffers
;
657 static void radv_amdgpu_cs_add_buffer(struct radeon_cmdbuf
*_cs
,
658 struct radeon_winsys_bo
*_bo
)
660 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(_cs
);
661 struct radv_amdgpu_winsys_bo
*bo
= radv_amdgpu_winsys_bo(_bo
);
663 if (cs
->status
!= VK_SUCCESS
)
666 if (bo
->is_virtual
) {
667 radv_amdgpu_cs_add_virtual_buffer(_cs
, _bo
);
671 if (bo
->base
.is_local
)
674 radv_amdgpu_cs_add_buffer_internal(cs
, bo
->bo_handle
, bo
->priority
);
677 static void radv_amdgpu_cs_execute_secondary(struct radeon_cmdbuf
*_parent
,
678 struct radeon_cmdbuf
*_child
)
680 struct radv_amdgpu_cs
*parent
= radv_amdgpu_cs(_parent
);
681 struct radv_amdgpu_cs
*child
= radv_amdgpu_cs(_child
);
683 if (parent
->status
!= VK_SUCCESS
|| child
->status
!= VK_SUCCESS
)
686 for (unsigned i
= 0; i
< child
->num_buffers
; ++i
) {
687 radv_amdgpu_cs_add_buffer_internal(parent
,
688 child
->handles
[i
].bo_handle
,
689 child
->handles
[i
].bo_priority
);
692 for (unsigned i
= 0; i
< child
->num_virtual_buffers
; ++i
) {
693 radv_amdgpu_cs_add_buffer(&parent
->base
, child
->virtual_buffers
[i
]);
696 if (parent
->ws
->use_ib_bos
) {
697 if (parent
->base
.cdw
+ 4 > parent
->base
.max_dw
)
698 radv_amdgpu_cs_grow(&parent
->base
, 4);
700 radeon_emit(&parent
->base
, PKT3(PKT3_INDIRECT_BUFFER_CIK
, 2, 0));
701 radeon_emit(&parent
->base
, child
->ib
.ib_mc_address
);
702 radeon_emit(&parent
->base
, child
->ib
.ib_mc_address
>> 32);
703 radeon_emit(&parent
->base
, child
->ib
.size
);
705 if (parent
->base
.cdw
+ child
->base
.cdw
> parent
->base
.max_dw
)
706 radv_amdgpu_cs_grow(&parent
->base
, child
->base
.cdw
);
708 memcpy(parent
->base
.buf
+ parent
->base
.cdw
, child
->base
.buf
, 4 * child
->base
.cdw
);
709 parent
->base
.cdw
+= child
->base
.cdw
;
714 radv_amdgpu_get_bo_list(struct radv_amdgpu_winsys
*ws
,
715 struct radeon_cmdbuf
**cs_array
,
717 struct radv_amdgpu_winsys_bo
**extra_bo_array
,
718 unsigned num_extra_bo
,
719 struct radeon_cmdbuf
*extra_cs
,
720 const struct radv_winsys_bo_list
*radv_bo_list
,
721 unsigned *rnum_handles
,
722 struct drm_amdgpu_bo_list_entry
**rhandles
)
724 struct drm_amdgpu_bo_list_entry
*handles
= NULL
;
725 unsigned num_handles
= 0;
727 if (ws
->debug_all_bos
) {
728 struct radv_amdgpu_winsys_bo
*bo
;
730 pthread_mutex_lock(&ws
->global_bo_list_lock
);
732 handles
= malloc(sizeof(handles
[0]) * ws
->num_buffers
);
734 pthread_mutex_unlock(&ws
->global_bo_list_lock
);
735 return VK_ERROR_OUT_OF_HOST_MEMORY
;
738 LIST_FOR_EACH_ENTRY(bo
, &ws
->global_bo_list
, global_list_item
) {
739 assert(num_handles
< ws
->num_buffers
);
740 handles
[num_handles
].bo_handle
= bo
->bo_handle
;
741 handles
[num_handles
].bo_priority
= bo
->priority
;
745 pthread_mutex_unlock(&ws
->global_bo_list_lock
);
746 } else if (count
== 1 && !num_extra_bo
&& !extra_cs
&& !radv_bo_list
&&
747 !radv_amdgpu_cs(cs_array
[0])->num_virtual_buffers
) {
748 struct radv_amdgpu_cs
*cs
= (struct radv_amdgpu_cs
*)cs_array
[0];
749 if (cs
->num_buffers
== 0)
752 handles
= malloc(sizeof(handles
[0]) * cs
->num_buffers
);
754 return VK_ERROR_OUT_OF_HOST_MEMORY
;
756 memcpy(handles
, cs
->handles
,
757 sizeof(handles
[0]) * cs
->num_buffers
);
758 num_handles
= cs
->num_buffers
;
760 unsigned total_buffer_count
= num_extra_bo
;
761 num_handles
= num_extra_bo
;
762 for (unsigned i
= 0; i
< count
; ++i
) {
763 struct radv_amdgpu_cs
*cs
= (struct radv_amdgpu_cs
*)cs_array
[i
];
764 total_buffer_count
+= cs
->num_buffers
;
765 for (unsigned j
= 0; j
< cs
->num_virtual_buffers
; ++j
)
766 total_buffer_count
+= radv_amdgpu_winsys_bo(cs
->virtual_buffers
[j
])->bo_count
;
770 total_buffer_count
+= ((struct radv_amdgpu_cs
*)extra_cs
)->num_buffers
;
774 total_buffer_count
+= radv_bo_list
->count
;
777 if (total_buffer_count
== 0)
780 handles
= malloc(sizeof(handles
[0]) * total_buffer_count
);
782 return VK_ERROR_OUT_OF_HOST_MEMORY
;
784 for (unsigned i
= 0; i
< num_extra_bo
; i
++) {
785 handles
[i
].bo_handle
= extra_bo_array
[i
]->bo_handle
;
786 handles
[i
].bo_priority
= extra_bo_array
[i
]->priority
;
789 for (unsigned i
= 0; i
< count
+ !!extra_cs
; ++i
) {
790 struct radv_amdgpu_cs
*cs
;
793 cs
= (struct radv_amdgpu_cs
*)extra_cs
;
795 cs
= (struct radv_amdgpu_cs
*)cs_array
[i
];
797 if (!cs
->num_buffers
)
800 if (num_handles
== 0 && !cs
->num_virtual_buffers
) {
801 memcpy(handles
, cs
->handles
, cs
->num_buffers
* sizeof(struct drm_amdgpu_bo_list_entry
));
802 num_handles
= cs
->num_buffers
;
805 int unique_bo_so_far
= num_handles
;
806 for (unsigned j
= 0; j
< cs
->num_buffers
; ++j
) {
808 for (unsigned k
= 0; k
< unique_bo_so_far
; ++k
) {
809 if (handles
[k
].bo_handle
== cs
->handles
[j
].bo_handle
) {
815 handles
[num_handles
] = cs
->handles
[j
];
819 for (unsigned j
= 0; j
< cs
->num_virtual_buffers
; ++j
) {
820 struct radv_amdgpu_winsys_bo
*virtual_bo
= radv_amdgpu_winsys_bo(cs
->virtual_buffers
[j
]);
821 for(unsigned k
= 0; k
< virtual_bo
->bo_count
; ++k
) {
822 struct radv_amdgpu_winsys_bo
*bo
= virtual_bo
->bos
[k
];
824 for (unsigned m
= 0; m
< num_handles
; ++m
) {
825 if (handles
[m
].bo_handle
== bo
->bo_handle
) {
831 handles
[num_handles
].bo_handle
= bo
->bo_handle
;
832 handles
[num_handles
].bo_priority
= bo
->priority
;
840 unsigned unique_bo_so_far
= num_handles
;
841 for (unsigned i
= 0; i
< radv_bo_list
->count
; ++i
) {
842 struct radv_amdgpu_winsys_bo
*bo
= radv_amdgpu_winsys_bo(radv_bo_list
->bos
[i
]);
844 for (unsigned j
= 0; j
< unique_bo_so_far
; ++j
) {
845 if (bo
->bo_handle
== handles
[j
].bo_handle
) {
851 handles
[num_handles
].bo_handle
= bo
->bo_handle
;
852 handles
[num_handles
].bo_priority
= bo
->priority
;
860 *rnum_handles
= num_handles
;
865 static struct amdgpu_cs_fence_info
radv_set_cs_fence(struct radv_amdgpu_ctx
*ctx
, int ip_type
, int ring
)
867 struct amdgpu_cs_fence_info ret
= {0};
868 if (ctx
->fence_map
) {
869 ret
.handle
= radv_amdgpu_winsys_bo(ctx
->fence_bo
)->bo
;
870 ret
.offset
= (ip_type
* MAX_RINGS_PER_TYPE
+ ring
) * sizeof(uint64_t);
875 static void radv_assign_last_submit(struct radv_amdgpu_ctx
*ctx
,
876 struct radv_amdgpu_cs_request
*request
)
878 radv_amdgpu_request_to_fence(ctx
,
879 &ctx
->last_submission
[request
->ip_type
][request
->ring
],
884 radv_amdgpu_winsys_cs_submit_chained(struct radeon_winsys_ctx
*_ctx
,
886 struct radv_winsys_sem_info
*sem_info
,
887 const struct radv_winsys_bo_list
*radv_bo_list
,
888 struct radeon_cmdbuf
**cs_array
,
890 struct radeon_cmdbuf
*initial_preamble_cs
,
891 struct radeon_cmdbuf
*continue_preamble_cs
,
892 struct radeon_winsys_fence
*_fence
)
894 struct radv_amdgpu_ctx
*ctx
= radv_amdgpu_ctx(_ctx
);
895 struct radv_amdgpu_fence
*fence
= (struct radv_amdgpu_fence
*)_fence
;
896 struct radv_amdgpu_cs
*cs0
= radv_amdgpu_cs(cs_array
[0]);
897 struct drm_amdgpu_bo_list_entry
*handles
= NULL
;
898 struct radv_amdgpu_cs_request request
= {0};
899 struct amdgpu_cs_ib_info ibs
[2];
900 unsigned number_of_ibs
= 1;
901 unsigned num_handles
= 0;
904 for (unsigned i
= cs_count
; i
--;) {
905 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(cs_array
[i
]);
907 if (cs
->is_chained
) {
908 *cs
->ib_size_ptr
-= 4;
909 cs
->is_chained
= false;
912 if (i
+ 1 < cs_count
) {
913 struct radv_amdgpu_cs
*next
= radv_amdgpu_cs(cs_array
[i
+ 1]);
914 assert(cs
->base
.cdw
+ 4 <= cs
->base
.max_dw
);
916 cs
->is_chained
= true;
917 *cs
->ib_size_ptr
+= 4;
919 cs
->base
.buf
[cs
->base
.cdw
+ 0] = PKT3(PKT3_INDIRECT_BUFFER_CIK
, 2, 0);
920 cs
->base
.buf
[cs
->base
.cdw
+ 1] = next
->ib
.ib_mc_address
;
921 cs
->base
.buf
[cs
->base
.cdw
+ 2] = next
->ib
.ib_mc_address
>> 32;
922 cs
->base
.buf
[cs
->base
.cdw
+ 3] = S_3F2_CHAIN(1) | S_3F2_VALID(1) | next
->ib
.size
;
926 /* Get the BO list. */
927 result
= radv_amdgpu_get_bo_list(cs0
->ws
, cs_array
, cs_count
, NULL
, 0,
928 initial_preamble_cs
, radv_bo_list
,
929 &num_handles
, &handles
);
930 if (result
!= VK_SUCCESS
)
933 /* Configure the CS request. */
934 if (initial_preamble_cs
) {
935 ibs
[0] = radv_amdgpu_cs(initial_preamble_cs
)->ib
;
942 request
.ip_type
= cs0
->hw_ip
;
943 request
.ring
= queue_idx
;
944 request
.number_of_ibs
= number_of_ibs
;
946 request
.handles
= handles
;
947 request
.num_handles
= num_handles
;
948 request
.fence_info
= radv_set_cs_fence(ctx
, cs0
->hw_ip
, queue_idx
);
951 result
= radv_amdgpu_cs_submit(ctx
, &request
, sem_info
);
953 free(request
.handles
);
955 if (result
!= VK_SUCCESS
)
959 radv_amdgpu_request_to_fence(ctx
, fence
, &request
);
961 radv_assign_last_submit(ctx
, &request
);
967 radv_amdgpu_winsys_cs_submit_fallback(struct radeon_winsys_ctx
*_ctx
,
969 struct radv_winsys_sem_info
*sem_info
,
970 const struct radv_winsys_bo_list
*radv_bo_list
,
971 struct radeon_cmdbuf
**cs_array
,
973 struct radeon_cmdbuf
*initial_preamble_cs
,
974 struct radeon_cmdbuf
*continue_preamble_cs
,
975 struct radeon_winsys_fence
*_fence
)
977 struct radv_amdgpu_ctx
*ctx
= radv_amdgpu_ctx(_ctx
);
978 struct radv_amdgpu_fence
*fence
= (struct radv_amdgpu_fence
*)_fence
;
979 struct drm_amdgpu_bo_list_entry
*handles
= NULL
;
980 struct radv_amdgpu_cs_request request
= {};
981 struct amdgpu_cs_ib_info
*ibs
;
982 struct radv_amdgpu_cs
*cs0
;
983 unsigned num_handles
= 0;
984 unsigned number_of_ibs
;
988 cs0
= radv_amdgpu_cs(cs_array
[0]);
990 /* Compute the number of IBs for this submit. */
991 number_of_ibs
= cs_count
+ !!initial_preamble_cs
;
993 /* Get the BO list. */
994 result
= radv_amdgpu_get_bo_list(cs0
->ws
, &cs_array
[0], cs_count
, NULL
, 0,
995 initial_preamble_cs
, radv_bo_list
,
996 &num_handles
, &handles
);
997 if (result
!= VK_SUCCESS
)
1000 ibs
= malloc(number_of_ibs
* sizeof(*ibs
));
1003 return VK_ERROR_OUT_OF_HOST_MEMORY
;
1006 /* Configure the CS request. */
1007 if (initial_preamble_cs
)
1008 ibs
[0] = radv_amdgpu_cs(initial_preamble_cs
)->ib
;
1010 for (unsigned i
= 0; i
< cs_count
; i
++) {
1011 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(cs_array
[i
]);
1013 ibs
[i
+ !!initial_preamble_cs
] = cs
->ib
;
1015 if (cs
->is_chained
) {
1016 *cs
->ib_size_ptr
-= 4;
1017 cs
->is_chained
= false;
1021 request
.ip_type
= cs0
->hw_ip
;
1022 request
.ring
= queue_idx
;
1023 request
.handles
= handles
;
1024 request
.num_handles
= num_handles
;
1025 request
.number_of_ibs
= number_of_ibs
;
1027 request
.fence_info
= radv_set_cs_fence(ctx
, cs0
->hw_ip
, queue_idx
);
1029 /* Submit the CS. */
1030 result
= radv_amdgpu_cs_submit(ctx
, &request
, sem_info
);
1032 free(request
.handles
);
1035 if (result
!= VK_SUCCESS
)
1039 radv_amdgpu_request_to_fence(ctx
, fence
, &request
);
1041 radv_assign_last_submit(ctx
, &request
);
1047 radv_amdgpu_winsys_cs_submit_sysmem(struct radeon_winsys_ctx
*_ctx
,
1049 struct radv_winsys_sem_info
*sem_info
,
1050 const struct radv_winsys_bo_list
*radv_bo_list
,
1051 struct radeon_cmdbuf
**cs_array
,
1053 struct radeon_cmdbuf
*initial_preamble_cs
,
1054 struct radeon_cmdbuf
*continue_preamble_cs
,
1055 struct radeon_winsys_fence
*_fence
)
1057 struct radv_amdgpu_ctx
*ctx
= radv_amdgpu_ctx(_ctx
);
1058 struct radv_amdgpu_fence
*fence
= (struct radv_amdgpu_fence
*)_fence
;
1059 struct radv_amdgpu_cs
*cs0
= radv_amdgpu_cs(cs_array
[0]);
1060 struct radeon_winsys
*ws
= (struct radeon_winsys
*)cs0
->ws
;
1061 struct radv_amdgpu_cs_request request
;
1062 uint32_t pad_word
= PKT3_NOP_PAD
;
1063 bool emit_signal_sem
= sem_info
->cs_emit_signal
;
1066 if (radv_amdgpu_winsys(ws
)->info
.chip_class
== GFX6
)
1067 pad_word
= 0x80000000;
1071 for (unsigned i
= 0; i
< cs_count
;) {
1072 struct amdgpu_cs_ib_info
*ibs
;
1073 struct radeon_winsys_bo
**bos
;
1074 struct radeon_cmdbuf
*preamble_cs
= i
? continue_preamble_cs
: initial_preamble_cs
;
1075 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(cs_array
[i
]);
1076 struct drm_amdgpu_bo_list_entry
*handles
= NULL
;
1077 unsigned num_handles
= 0;
1078 unsigned number_of_ibs
;
1082 unsigned pad_words
= 0;
1084 /* Compute the number of IBs for this submit. */
1085 number_of_ibs
= cs
->num_old_cs_buffers
+ 1;
1087 ibs
= malloc(number_of_ibs
* sizeof(*ibs
));
1089 return VK_ERROR_OUT_OF_HOST_MEMORY
;
1091 bos
= malloc(number_of_ibs
* sizeof(*bos
));
1094 return VK_ERROR_OUT_OF_HOST_MEMORY
;
1097 if (number_of_ibs
> 1) {
1098 /* Special path when the maximum size in dwords has
1099 * been reached because we need to handle more than one
1102 struct radeon_cmdbuf
**new_cs_array
;
1105 new_cs_array
= malloc(cs
->num_old_cs_buffers
*
1106 sizeof(*new_cs_array
));
1107 assert(new_cs_array
);
1109 for (unsigned j
= 0; j
< cs
->num_old_cs_buffers
; j
++)
1110 new_cs_array
[idx
++] = &cs
->old_cs_buffers
[j
];
1111 new_cs_array
[idx
++] = cs_array
[i
];
1113 for (unsigned j
= 0; j
< number_of_ibs
; j
++) {
1114 struct radeon_cmdbuf
*rcs
= new_cs_array
[j
];
1115 bool needs_preamble
= preamble_cs
&& j
== 0;
1119 size
+= preamble_cs
->cdw
;
1122 assert(size
< 0xffff8);
1124 while (!size
|| (size
& 7)) {
1129 bos
[j
] = ws
->buffer_create(ws
, 4 * size
, 4096,
1131 RADEON_FLAG_CPU_ACCESS
|
1132 RADEON_FLAG_NO_INTERPROCESS_SHARING
|
1133 RADEON_FLAG_READ_ONLY
,
1134 RADV_BO_PRIORITY_CS
);
1135 ptr
= ws
->buffer_map(bos
[j
]);
1137 if (needs_preamble
) {
1138 memcpy(ptr
, preamble_cs
->buf
, preamble_cs
->cdw
* 4);
1139 ptr
+= preamble_cs
->cdw
;
1142 memcpy(ptr
, rcs
->buf
, 4 * rcs
->cdw
);
1145 for (unsigned k
= 0; k
< pad_words
; ++k
)
1149 ibs
[j
].ib_mc_address
= radv_buffer_get_va(bos
[j
]);
1157 size
+= preamble_cs
->cdw
;
1159 while (i
+ cnt
< cs_count
&& 0xffff8 - size
>= radv_amdgpu_cs(cs_array
[i
+ cnt
])->base
.cdw
) {
1160 size
+= radv_amdgpu_cs(cs_array
[i
+ cnt
])->base
.cdw
;
1164 while (!size
|| (size
& 7)) {
1170 bos
[0] = ws
->buffer_create(ws
, 4 * size
, 4096,
1172 RADEON_FLAG_CPU_ACCESS
|
1173 RADEON_FLAG_NO_INTERPROCESS_SHARING
|
1174 RADEON_FLAG_READ_ONLY
,
1175 RADV_BO_PRIORITY_CS
);
1176 ptr
= ws
->buffer_map(bos
[0]);
1179 memcpy(ptr
, preamble_cs
->buf
, preamble_cs
->cdw
* 4);
1180 ptr
+= preamble_cs
->cdw
;
1183 for (unsigned j
= 0; j
< cnt
; ++j
) {
1184 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(cs_array
[i
+ j
]);
1185 memcpy(ptr
, cs
->base
.buf
, 4 * cs
->base
.cdw
);
1186 ptr
+= cs
->base
.cdw
;
1190 for (unsigned j
= 0; j
< pad_words
; ++j
)
1194 ibs
[0].ib_mc_address
= radv_buffer_get_va(bos
[0]);
1198 result
= radv_amdgpu_get_bo_list(cs0
->ws
, &cs_array
[i
], cnt
,
1199 (struct radv_amdgpu_winsys_bo
**)bos
,
1200 number_of_ibs
, preamble_cs
,
1202 &num_handles
, &handles
);
1203 if (result
!= VK_SUCCESS
) {
1209 memset(&request
, 0, sizeof(request
));
1211 request
.ip_type
= cs0
->hw_ip
;
1212 request
.ring
= queue_idx
;
1213 request
.handles
= handles
;
1214 request
.num_handles
= num_handles
;
1215 request
.number_of_ibs
= number_of_ibs
;
1217 request
.fence_info
= radv_set_cs_fence(ctx
, cs0
->hw_ip
, queue_idx
);
1219 sem_info
->cs_emit_signal
= (i
== cs_count
- cnt
) ? emit_signal_sem
: false;
1220 result
= radv_amdgpu_cs_submit(ctx
, &request
, sem_info
);
1222 free(request
.handles
);
1224 for (unsigned j
= 0; j
< number_of_ibs
; j
++) {
1225 ws
->buffer_destroy(bos
[j
]);
1231 if (result
!= VK_SUCCESS
)
1237 radv_amdgpu_request_to_fence(ctx
, fence
, &request
);
1239 radv_assign_last_submit(ctx
, &request
);
1244 static VkResult
radv_amdgpu_winsys_cs_submit(struct radeon_winsys_ctx
*_ctx
,
1246 struct radeon_cmdbuf
**cs_array
,
1248 struct radeon_cmdbuf
*initial_preamble_cs
,
1249 struct radeon_cmdbuf
*continue_preamble_cs
,
1250 struct radv_winsys_sem_info
*sem_info
,
1251 const struct radv_winsys_bo_list
*bo_list
,
1253 struct radeon_winsys_fence
*_fence
)
1255 struct radv_amdgpu_cs
*cs
= radv_amdgpu_cs(cs_array
[0]);
1256 struct radv_amdgpu_ctx
*ctx
= radv_amdgpu_ctx(_ctx
);
1260 if (!cs
->ws
->use_ib_bos
) {
1261 result
= radv_amdgpu_winsys_cs_submit_sysmem(_ctx
, queue_idx
, sem_info
, bo_list
, cs_array
,
1262 cs_count
, initial_preamble_cs
, continue_preamble_cs
, _fence
);
1263 } else if (can_patch
) {
1264 result
= radv_amdgpu_winsys_cs_submit_chained(_ctx
, queue_idx
, sem_info
, bo_list
, cs_array
,
1265 cs_count
, initial_preamble_cs
, continue_preamble_cs
, _fence
);
1267 result
= radv_amdgpu_winsys_cs_submit_fallback(_ctx
, queue_idx
, sem_info
, bo_list
, cs_array
,
1268 cs_count
, initial_preamble_cs
, continue_preamble_cs
, _fence
);
1271 radv_amdgpu_signal_sems(ctx
, cs
->hw_ip
, queue_idx
, sem_info
);
1275 static void *radv_amdgpu_winsys_get_cpu_addr(void *_cs
, uint64_t addr
)
1277 struct radv_amdgpu_cs
*cs
= (struct radv_amdgpu_cs
*)_cs
;
1282 for (unsigned i
= 0; i
<= cs
->num_old_ib_buffers
; ++i
) {
1283 struct radv_amdgpu_winsys_bo
*bo
;
1285 bo
= (struct radv_amdgpu_winsys_bo
*)
1286 (i
== cs
->num_old_ib_buffers
? cs
->ib_buffer
: cs
->old_ib_buffers
[i
]);
1287 if (addr
>= bo
->base
.va
&& addr
- bo
->base
.va
< bo
->size
) {
1288 if (amdgpu_bo_cpu_map(bo
->bo
, &ret
) == 0)
1289 return (char *)ret
+ (addr
- bo
->base
.va
);
1292 if(cs
->ws
->debug_all_bos
) {
1293 pthread_mutex_lock(&cs
->ws
->global_bo_list_lock
);
1294 list_for_each_entry(struct radv_amdgpu_winsys_bo
, bo
,
1295 &cs
->ws
->global_bo_list
, global_list_item
) {
1296 if (addr
>= bo
->base
.va
&& addr
- bo
->base
.va
< bo
->size
) {
1297 if (amdgpu_bo_cpu_map(bo
->bo
, &ret
) == 0) {
1298 pthread_mutex_unlock(&cs
->ws
->global_bo_list_lock
);
1299 return (char *)ret
+ (addr
- bo
->base
.va
);
1303 pthread_mutex_unlock(&cs
->ws
->global_bo_list_lock
);
1308 static void radv_amdgpu_winsys_cs_dump(struct radeon_cmdbuf
*_cs
,
1310 const int *trace_ids
, int trace_id_count
)
1312 struct radv_amdgpu_cs
*cs
= (struct radv_amdgpu_cs
*)_cs
;
1313 void *ib
= cs
->base
.buf
;
1314 int num_dw
= cs
->base
.cdw
;
1316 if (cs
->ws
->use_ib_bos
) {
1317 ib
= radv_amdgpu_winsys_get_cpu_addr(cs
, cs
->ib
.ib_mc_address
);
1318 num_dw
= cs
->ib
.size
;
1321 ac_parse_ib(file
, ib
, num_dw
, trace_ids
, trace_id_count
, "main IB",
1322 cs
->ws
->info
.chip_class
, radv_amdgpu_winsys_get_cpu_addr
, cs
);
1325 static uint32_t radv_to_amdgpu_priority(enum radeon_ctx_priority radv_priority
)
1327 switch (radv_priority
) {
1328 case RADEON_CTX_PRIORITY_REALTIME
:
1329 return AMDGPU_CTX_PRIORITY_VERY_HIGH
;
1330 case RADEON_CTX_PRIORITY_HIGH
:
1331 return AMDGPU_CTX_PRIORITY_HIGH
;
1332 case RADEON_CTX_PRIORITY_MEDIUM
:
1333 return AMDGPU_CTX_PRIORITY_NORMAL
;
1334 case RADEON_CTX_PRIORITY_LOW
:
1335 return AMDGPU_CTX_PRIORITY_LOW
;
1337 unreachable("Invalid context priority");
1341 static VkResult
radv_amdgpu_ctx_create(struct radeon_winsys
*_ws
,
1342 enum radeon_ctx_priority priority
,
1343 struct radeon_winsys_ctx
**rctx
)
1345 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1346 struct radv_amdgpu_ctx
*ctx
= CALLOC_STRUCT(radv_amdgpu_ctx
);
1347 uint32_t amdgpu_priority
= radv_to_amdgpu_priority(priority
);
1352 return VK_ERROR_OUT_OF_HOST_MEMORY
;
1354 r
= amdgpu_cs_ctx_create2(ws
->dev
, amdgpu_priority
, &ctx
->ctx
);
1355 if (r
&& r
== -EACCES
) {
1356 result
= VK_ERROR_NOT_PERMITTED_EXT
;
1359 fprintf(stderr
, "amdgpu: radv_amdgpu_cs_ctx_create2 failed. (%i)\n", r
);
1360 result
= VK_ERROR_OUT_OF_HOST_MEMORY
;
1365 assert(AMDGPU_HW_IP_NUM
* MAX_RINGS_PER_TYPE
* sizeof(uint64_t) <= 4096);
1366 ctx
->fence_bo
= ws
->base
.buffer_create(&ws
->base
, 4096, 8,
1368 RADEON_FLAG_CPU_ACCESS
|
1369 RADEON_FLAG_NO_INTERPROCESS_SHARING
,
1370 RADV_BO_PRIORITY_CS
);
1371 if (!ctx
->fence_bo
) {
1372 result
= VK_ERROR_OUT_OF_DEVICE_MEMORY
;
1376 ctx
->fence_map
= (uint64_t *)ws
->base
.buffer_map(ctx
->fence_bo
);
1377 if (!ctx
->fence_map
) {
1378 result
= VK_ERROR_OUT_OF_DEVICE_MEMORY
;
1382 memset(ctx
->fence_map
, 0, 4096);
1384 *rctx
= (struct radeon_winsys_ctx
*)ctx
;
1388 ws
->base
.buffer_destroy(ctx
->fence_bo
);
1390 amdgpu_cs_ctx_free(ctx
->ctx
);
1396 static void radv_amdgpu_ctx_destroy(struct radeon_winsys_ctx
*rwctx
)
1398 struct radv_amdgpu_ctx
*ctx
= (struct radv_amdgpu_ctx
*)rwctx
;
1399 ctx
->ws
->base
.buffer_destroy(ctx
->fence_bo
);
1400 amdgpu_cs_ctx_free(ctx
->ctx
);
1404 static bool radv_amdgpu_ctx_wait_idle(struct radeon_winsys_ctx
*rwctx
,
1405 enum ring_type ring_type
, int ring_index
)
1407 struct radv_amdgpu_ctx
*ctx
= (struct radv_amdgpu_ctx
*)rwctx
;
1408 int ip_type
= ring_to_hw_ip(ring_type
);
1410 if (ctx
->last_submission
[ip_type
][ring_index
].fence
.fence
) {
1412 int ret
= amdgpu_cs_query_fence_status(&ctx
->last_submission
[ip_type
][ring_index
].fence
,
1413 1000000000ull, 0, &expired
);
1415 if (ret
|| !expired
)
1422 static struct radeon_winsys_sem
*radv_amdgpu_create_sem(struct radeon_winsys
*_ws
)
1424 struct amdgpu_cs_fence
*sem
= CALLOC_STRUCT(amdgpu_cs_fence
);
1428 return (struct radeon_winsys_sem
*)sem
;
1431 static void radv_amdgpu_destroy_sem(struct radeon_winsys_sem
*_sem
)
1433 struct amdgpu_cs_fence
*sem
= (struct amdgpu_cs_fence
*)_sem
;
1437 static int radv_amdgpu_signal_sems(struct radv_amdgpu_ctx
*ctx
,
1440 struct radv_winsys_sem_info
*sem_info
)
1442 for (unsigned i
= 0; i
< sem_info
->signal
.sem_count
; i
++) {
1443 struct amdgpu_cs_fence
*sem
= (struct amdgpu_cs_fence
*)(sem_info
->signal
.sem
)[i
];
1448 *sem
= ctx
->last_submission
[ip_type
][ring
].fence
;
1453 static void *radv_amdgpu_cs_alloc_syncobj_chunk(struct radv_winsys_sem_counts
*counts
,
1454 const uint32_t *syncobj_override
,
1455 struct drm_amdgpu_cs_chunk
*chunk
, int chunk_id
)
1457 const uint32_t *src
= syncobj_override
? syncobj_override
: counts
->syncobj
;
1458 struct drm_amdgpu_cs_chunk_sem
*syncobj
= malloc(sizeof(struct drm_amdgpu_cs_chunk_sem
) * counts
->syncobj_count
);
1462 for (unsigned i
= 0; i
< counts
->syncobj_count
; i
++) {
1463 struct drm_amdgpu_cs_chunk_sem
*sem
= &syncobj
[i
];
1464 sem
->handle
= src
[i
];
1467 chunk
->chunk_id
= chunk_id
;
1468 chunk
->length_dw
= sizeof(struct drm_amdgpu_cs_chunk_sem
) / 4 * counts
->syncobj_count
;
1469 chunk
->chunk_data
= (uint64_t)(uintptr_t)syncobj
;
1474 radv_amdgpu_cs_alloc_timeline_syncobj_chunk(struct radv_winsys_sem_counts
*counts
,
1475 const uint32_t *syncobj_override
,
1476 struct drm_amdgpu_cs_chunk
*chunk
, int chunk_id
)
1478 const uint32_t *src
= syncobj_override
? syncobj_override
: counts
->syncobj
;
1479 struct drm_amdgpu_cs_chunk_syncobj
*syncobj
= malloc(sizeof(struct drm_amdgpu_cs_chunk_syncobj
) *
1480 (counts
->syncobj_count
+ counts
->timeline_syncobj_count
));
1484 for (unsigned i
= 0; i
< counts
->syncobj_count
; i
++) {
1485 struct drm_amdgpu_cs_chunk_syncobj
*sem
= &syncobj
[i
];
1486 sem
->handle
= src
[i
];
1491 for (unsigned i
= 0; i
< counts
->timeline_syncobj_count
; i
++) {
1492 struct drm_amdgpu_cs_chunk_syncobj
*sem
= &syncobj
[i
+ counts
->syncobj_count
];
1493 sem
->handle
= counts
->syncobj
[i
+ counts
->syncobj_count
];
1494 sem
->flags
= DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT
;
1495 sem
->point
= counts
->points
[i
];
1498 chunk
->chunk_id
= chunk_id
;
1499 chunk
->length_dw
= sizeof(struct drm_amdgpu_cs_chunk_syncobj
) / 4 *
1500 (counts
->syncobj_count
+ counts
->timeline_syncobj_count
);
1501 chunk
->chunk_data
= (uint64_t)(uintptr_t)syncobj
;
1505 static int radv_amdgpu_cache_alloc_syncobjs(struct radv_amdgpu_winsys
*ws
, unsigned count
, uint32_t *dst
)
1507 pthread_mutex_lock(&ws
->syncobj_lock
);
1508 if (count
> ws
->syncobj_capacity
) {
1509 if (ws
->syncobj_capacity
> UINT32_MAX
/ 2)
1512 unsigned new_capacity
= MAX2(count
, ws
->syncobj_capacity
* 2);
1513 uint32_t *n
= realloc(ws
->syncobj
, new_capacity
* sizeof(*ws
->syncobj
));
1516 ws
->syncobj_capacity
= new_capacity
;
1520 while(ws
->syncobj_count
< count
) {
1521 int r
= amdgpu_cs_create_syncobj(ws
->dev
, ws
->syncobj
+ ws
->syncobj_count
);
1524 ++ws
->syncobj_count
;
1527 for (unsigned i
= 0; i
< count
; ++i
)
1528 dst
[i
] = ws
->syncobj
[--ws
->syncobj_count
];
1530 pthread_mutex_unlock(&ws
->syncobj_lock
);
1534 pthread_mutex_unlock(&ws
->syncobj_lock
);
1538 static void radv_amdgpu_cache_free_syncobjs(struct radv_amdgpu_winsys
*ws
, unsigned count
, uint32_t *src
)
1540 pthread_mutex_lock(&ws
->syncobj_lock
);
1542 uint32_t cache_count
= MIN2(count
, UINT32_MAX
- ws
->syncobj_count
);
1543 if (cache_count
+ ws
->syncobj_count
> ws
->syncobj_capacity
) {
1544 unsigned new_capacity
= MAX2(ws
->syncobj_count
+ cache_count
, ws
->syncobj_capacity
* 2);
1545 uint32_t* n
= realloc(ws
->syncobj
, new_capacity
* sizeof(*ws
->syncobj
));
1547 ws
->syncobj_capacity
= new_capacity
;
1552 for (unsigned i
= 0; i
< count
; ++i
) {
1553 if (ws
->syncobj_count
< ws
->syncobj_capacity
)
1554 ws
->syncobj
[ws
->syncobj_count
++] = src
[i
];
1556 amdgpu_cs_destroy_syncobj(ws
->dev
, src
[i
]);
1559 pthread_mutex_unlock(&ws
->syncobj_lock
);
1563 static int radv_amdgpu_cs_prepare_syncobjs(struct radv_amdgpu_winsys
*ws
,
1564 struct radv_winsys_sem_counts
*counts
,
1565 uint32_t **out_syncobjs
)
1569 if (!ws
->info
.has_timeline_syncobj
|| !counts
->syncobj_count
) {
1570 *out_syncobjs
= NULL
;
1574 *out_syncobjs
= malloc(counts
->syncobj_count
* sizeof(**out_syncobjs
));
1578 r
= radv_amdgpu_cache_alloc_syncobjs(ws
, counts
->syncobj_count
, *out_syncobjs
);
1582 for (unsigned i
= 0; i
< counts
->syncobj_count
; ++i
) {
1583 r
= amdgpu_cs_syncobj_transfer(ws
->dev
, (*out_syncobjs
)[i
], 0, counts
->syncobj
[i
], 0, DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT
);
1588 r
= amdgpu_cs_syncobj_reset(ws
->dev
, counts
->syncobj
, counts
->syncobj_reset_count
);
1594 radv_amdgpu_cache_free_syncobjs(ws
, counts
->syncobj_count
, *out_syncobjs
);
1595 free(*out_syncobjs
);
1596 *out_syncobjs
= NULL
;
1601 radv_amdgpu_cs_submit(struct radv_amdgpu_ctx
*ctx
,
1602 struct radv_amdgpu_cs_request
*request
,
1603 struct radv_winsys_sem_info
*sem_info
)
1609 struct drm_amdgpu_cs_chunk
*chunks
;
1610 struct drm_amdgpu_cs_chunk_data
*chunk_data
;
1611 struct drm_amdgpu_cs_chunk_dep
*sem_dependencies
= NULL
;
1612 bool use_bo_list_create
= ctx
->ws
->info
.drm_minor
< 27;
1613 struct drm_amdgpu_bo_list_in bo_list_in
;
1614 void *wait_syncobj
= NULL
, *signal_syncobj
= NULL
;
1615 uint32_t *in_syncobjs
= NULL
;
1617 struct amdgpu_cs_fence
*sem
;
1618 uint32_t bo_list
= 0;
1619 VkResult result
= VK_SUCCESS
;
1621 user_fence
= (request
->fence_info
.handle
!= NULL
);
1622 size
= request
->number_of_ibs
+ (user_fence
? 2 : 1) + (!use_bo_list_create
? 1 : 0) + 3;
1624 chunks
= malloc(sizeof(chunks
[0]) * size
);
1626 return VK_ERROR_OUT_OF_HOST_MEMORY
;
1628 size
= request
->number_of_ibs
+ (user_fence
? 1 : 0);
1630 chunk_data
= malloc(sizeof(chunk_data
[0]) * size
);
1632 result
= VK_ERROR_OUT_OF_HOST_MEMORY
;
1636 num_chunks
= request
->number_of_ibs
;
1637 for (i
= 0; i
< request
->number_of_ibs
; i
++) {
1638 struct amdgpu_cs_ib_info
*ib
;
1639 chunks
[i
].chunk_id
= AMDGPU_CHUNK_ID_IB
;
1640 chunks
[i
].length_dw
= sizeof(struct drm_amdgpu_cs_chunk_ib
) / 4;
1641 chunks
[i
].chunk_data
= (uint64_t)(uintptr_t)&chunk_data
[i
];
1643 ib
= &request
->ibs
[i
];
1645 chunk_data
[i
].ib_data
._pad
= 0;
1646 chunk_data
[i
].ib_data
.va_start
= ib
->ib_mc_address
;
1647 chunk_data
[i
].ib_data
.ib_bytes
= ib
->size
* 4;
1648 chunk_data
[i
].ib_data
.ip_type
= request
->ip_type
;
1649 chunk_data
[i
].ib_data
.ip_instance
= request
->ip_instance
;
1650 chunk_data
[i
].ib_data
.ring
= request
->ring
;
1651 chunk_data
[i
].ib_data
.flags
= ib
->flags
;
1657 chunks
[i
].chunk_id
= AMDGPU_CHUNK_ID_FENCE
;
1658 chunks
[i
].length_dw
= sizeof(struct drm_amdgpu_cs_chunk_fence
) / 4;
1659 chunks
[i
].chunk_data
= (uint64_t)(uintptr_t)&chunk_data
[i
];
1661 amdgpu_cs_chunk_fence_info_to_data(&request
->fence_info
,
1665 if ((sem_info
->wait
.syncobj_count
|| sem_info
->wait
.timeline_syncobj_count
) && sem_info
->cs_emit_wait
) {
1666 r
= radv_amdgpu_cs_prepare_syncobjs(ctx
->ws
, &sem_info
->wait
, &in_syncobjs
);
1670 if (ctx
->ws
->info
.has_timeline_syncobj
) {
1671 wait_syncobj
= radv_amdgpu_cs_alloc_timeline_syncobj_chunk(&sem_info
->wait
,
1673 &chunks
[num_chunks
],
1674 AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT
);
1676 wait_syncobj
= radv_amdgpu_cs_alloc_syncobj_chunk(&sem_info
->wait
,
1678 &chunks
[num_chunks
],
1679 AMDGPU_CHUNK_ID_SYNCOBJ_IN
);
1681 if (!wait_syncobj
) {
1682 result
= VK_ERROR_OUT_OF_HOST_MEMORY
;
1687 if (sem_info
->wait
.sem_count
== 0)
1688 sem_info
->cs_emit_wait
= false;
1692 if (sem_info
->wait
.sem_count
&& sem_info
->cs_emit_wait
) {
1693 sem_dependencies
= malloc(sizeof(sem_dependencies
[0]) * sem_info
->wait
.sem_count
);
1694 if (!sem_dependencies
) {
1695 result
= VK_ERROR_OUT_OF_HOST_MEMORY
;
1701 for (unsigned j
= 0; j
< sem_info
->wait
.sem_count
; j
++) {
1702 sem
= (struct amdgpu_cs_fence
*)sem_info
->wait
.sem
[j
];
1705 struct drm_amdgpu_cs_chunk_dep
*dep
= &sem_dependencies
[sem_count
++];
1707 amdgpu_cs_chunk_fence_to_dep(sem
, dep
);
1709 sem
->context
= NULL
;
1713 /* dependencies chunk */
1714 chunks
[i
].chunk_id
= AMDGPU_CHUNK_ID_DEPENDENCIES
;
1715 chunks
[i
].length_dw
= sizeof(struct drm_amdgpu_cs_chunk_dep
) / 4 * sem_count
;
1716 chunks
[i
].chunk_data
= (uint64_t)(uintptr_t)sem_dependencies
;
1718 sem_info
->cs_emit_wait
= false;
1721 if ((sem_info
->signal
.syncobj_count
|| sem_info
->signal
.timeline_syncobj_count
) && sem_info
->cs_emit_signal
) {
1722 if (ctx
->ws
->info
.has_timeline_syncobj
) {
1723 signal_syncobj
= radv_amdgpu_cs_alloc_timeline_syncobj_chunk(&sem_info
->signal
,
1725 &chunks
[num_chunks
],
1726 AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL
);
1728 signal_syncobj
= radv_amdgpu_cs_alloc_syncobj_chunk(&sem_info
->signal
,
1730 &chunks
[num_chunks
],
1731 AMDGPU_CHUNK_ID_SYNCOBJ_OUT
);
1733 if (!signal_syncobj
) {
1734 result
= VK_ERROR_OUT_OF_HOST_MEMORY
;
1740 if (use_bo_list_create
) {
1741 /* Legacy path creating the buffer list handle and passing it
1744 r
= amdgpu_bo_list_create_raw(ctx
->ws
->dev
, request
->num_handles
,
1745 request
->handles
, &bo_list
);
1748 fprintf(stderr
, "amdgpu: Not enough memory for buffer list creation.\n");
1749 result
= VK_ERROR_OUT_OF_HOST_MEMORY
;
1751 fprintf(stderr
, "amdgpu: buffer list creation failed (%d).\n", r
);
1752 result
= VK_ERROR_UNKNOWN
;
1757 /* Standard path passing the buffer list via the CS ioctl. */
1758 bo_list_in
.operation
= ~0;
1759 bo_list_in
.list_handle
= ~0;
1760 bo_list_in
.bo_number
= request
->num_handles
;
1761 bo_list_in
.bo_info_size
= sizeof(struct drm_amdgpu_bo_list_entry
);
1762 bo_list_in
.bo_info_ptr
= (uint64_t)(uintptr_t)request
->handles
;
1764 chunks
[num_chunks
].chunk_id
= AMDGPU_CHUNK_ID_BO_HANDLES
;
1765 chunks
[num_chunks
].length_dw
= sizeof(struct drm_amdgpu_bo_list_in
) / 4;
1766 chunks
[num_chunks
].chunk_data
= (uintptr_t)&bo_list_in
;
1770 r
= amdgpu_cs_submit_raw2(ctx
->ws
->dev
,
1778 amdgpu_bo_list_destroy_raw(ctx
->ws
->dev
, bo_list
);
1782 fprintf(stderr
, "amdgpu: Not enough memory for command submission.\n");
1783 result
= VK_ERROR_OUT_OF_HOST_MEMORY
;
1784 } else if (r
== -ECANCELED
) {
1785 fprintf(stderr
, "amdgpu: The CS has been cancelled because the context is lost.\n");
1786 result
= VK_ERROR_DEVICE_LOST
;
1788 fprintf(stderr
, "amdgpu: The CS has been rejected, "
1789 "see dmesg for more information (%i).\n", r
);
1790 result
= VK_ERROR_UNKNOWN
;
1796 radv_amdgpu_cache_free_syncobjs(ctx
->ws
, sem_info
->wait
.syncobj_count
, in_syncobjs
);
1801 free(sem_dependencies
);
1803 free(signal_syncobj
);
1807 static int radv_amdgpu_create_syncobj(struct radeon_winsys
*_ws
,
1808 bool create_signaled
,
1811 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1814 if (create_signaled
)
1815 flags
|= DRM_SYNCOBJ_CREATE_SIGNALED
;
1817 return amdgpu_cs_create_syncobj2(ws
->dev
, flags
, handle
);
1820 static void radv_amdgpu_destroy_syncobj(struct radeon_winsys
*_ws
,
1823 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1824 amdgpu_cs_destroy_syncobj(ws
->dev
, handle
);
1827 static void radv_amdgpu_reset_syncobj(struct radeon_winsys
*_ws
,
1830 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1831 amdgpu_cs_syncobj_reset(ws
->dev
, &handle
, 1);
1834 static void radv_amdgpu_signal_syncobj(struct radeon_winsys
*_ws
,
1835 uint32_t handle
, uint64_t point
)
1837 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1839 amdgpu_cs_syncobj_timeline_signal(ws
->dev
, &handle
, &point
, 1);
1841 amdgpu_cs_syncobj_signal(ws
->dev
, &handle
, 1);
1844 static VkResult
radv_amdgpu_query_syncobj(struct radeon_winsys
*_ws
,
1845 uint32_t handle
, uint64_t *point
)
1847 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1848 int ret
= amdgpu_cs_syncobj_query(ws
->dev
, &handle
, point
, 1);
1851 else if (ret
== -ENOMEM
)
1852 return VK_ERROR_OUT_OF_HOST_MEMORY
;
1854 /* Remaining error are driver internal issues: EFAULT for
1855 * dangling pointers and ENOENT for non-existing syncobj. */
1856 fprintf(stderr
, "amdgpu: internal error in radv_amdgpu_query_syncobj. (%d)\n", ret
);
1857 return VK_ERROR_UNKNOWN
;
1861 static bool radv_amdgpu_wait_syncobj(struct radeon_winsys
*_ws
, const uint32_t *handles
,
1862 uint32_t handle_count
, bool wait_all
, uint64_t timeout
)
1864 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1867 /* The timeouts are signed, while vulkan timeouts are unsigned. */
1868 timeout
= MIN2(timeout
, INT64_MAX
);
1870 int ret
= amdgpu_cs_syncobj_wait(ws
->dev
, (uint32_t*)handles
, handle_count
, timeout
,
1871 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT
|
1872 (wait_all
? DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL
: 0),
1876 } else if (ret
== -ETIME
) {
1879 fprintf(stderr
, "amdgpu: radv_amdgpu_wait_syncobj failed!\nerrno: %d\n", errno
);
1884 static bool radv_amdgpu_wait_timeline_syncobj(struct radeon_winsys
*_ws
, const uint32_t *handles
,
1885 const uint64_t *points
, uint32_t handle_count
,
1886 bool wait_all
, bool available
, uint64_t timeout
)
1888 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1890 /* The timeouts are signed, while vulkan timeouts are unsigned. */
1891 timeout
= MIN2(timeout
, INT64_MAX
);
1893 int ret
= amdgpu_cs_syncobj_timeline_wait(ws
->dev
, (uint32_t*)handles
, (uint64_t*)points
,
1894 handle_count
, timeout
,
1895 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT
|
1896 (wait_all
? DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL
: 0) |
1897 (available
? DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE
: 0),
1901 } else if (ret
== -ETIME
) {
1904 fprintf(stderr
, "amdgpu: radv_amdgpu_wait_syncobj failed! (%d)\n", errno
);
1910 static int radv_amdgpu_export_syncobj(struct radeon_winsys
*_ws
,
1914 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1916 return amdgpu_cs_export_syncobj(ws
->dev
, syncobj
, fd
);
1919 static int radv_amdgpu_import_syncobj(struct radeon_winsys
*_ws
,
1923 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1925 return amdgpu_cs_import_syncobj(ws
->dev
, fd
, syncobj
);
1929 static int radv_amdgpu_export_syncobj_to_sync_file(struct radeon_winsys
*_ws
,
1933 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1935 return amdgpu_cs_syncobj_export_sync_file(ws
->dev
, syncobj
, fd
);
1938 static int radv_amdgpu_import_syncobj_from_sync_file(struct radeon_winsys
*_ws
,
1942 struct radv_amdgpu_winsys
*ws
= radv_amdgpu_winsys(_ws
);
1944 return amdgpu_cs_syncobj_import_sync_file(ws
->dev
, syncobj
, fd
);
1947 void radv_amdgpu_cs_init_functions(struct radv_amdgpu_winsys
*ws
)
1949 ws
->base
.ctx_create
= radv_amdgpu_ctx_create
;
1950 ws
->base
.ctx_destroy
= radv_amdgpu_ctx_destroy
;
1951 ws
->base
.ctx_wait_idle
= radv_amdgpu_ctx_wait_idle
;
1952 ws
->base
.cs_create
= radv_amdgpu_cs_create
;
1953 ws
->base
.cs_destroy
= radv_amdgpu_cs_destroy
;
1954 ws
->base
.cs_grow
= radv_amdgpu_cs_grow
;
1955 ws
->base
.cs_finalize
= radv_amdgpu_cs_finalize
;
1956 ws
->base
.cs_reset
= radv_amdgpu_cs_reset
;
1957 ws
->base
.cs_add_buffer
= radv_amdgpu_cs_add_buffer
;
1958 ws
->base
.cs_execute_secondary
= radv_amdgpu_cs_execute_secondary
;
1959 ws
->base
.cs_submit
= radv_amdgpu_winsys_cs_submit
;
1960 ws
->base
.cs_dump
= radv_amdgpu_winsys_cs_dump
;
1961 ws
->base
.create_fence
= radv_amdgpu_create_fence
;
1962 ws
->base
.destroy_fence
= radv_amdgpu_destroy_fence
;
1963 ws
->base
.reset_fence
= radv_amdgpu_reset_fence
;
1964 ws
->base
.signal_fence
= radv_amdgpu_signal_fence
;
1965 ws
->base
.is_fence_waitable
= radv_amdgpu_is_fence_waitable
;
1966 ws
->base
.create_sem
= radv_amdgpu_create_sem
;
1967 ws
->base
.destroy_sem
= radv_amdgpu_destroy_sem
;
1968 ws
->base
.create_syncobj
= radv_amdgpu_create_syncobj
;
1969 ws
->base
.destroy_syncobj
= radv_amdgpu_destroy_syncobj
;
1970 ws
->base
.reset_syncobj
= radv_amdgpu_reset_syncobj
;
1971 ws
->base
.signal_syncobj
= radv_amdgpu_signal_syncobj
;
1972 ws
->base
.query_syncobj
= radv_amdgpu_query_syncobj
;
1973 ws
->base
.wait_syncobj
= radv_amdgpu_wait_syncobj
;
1974 ws
->base
.wait_timeline_syncobj
= radv_amdgpu_wait_timeline_syncobj
;
1975 ws
->base
.export_syncobj
= radv_amdgpu_export_syncobj
;
1976 ws
->base
.import_syncobj
= radv_amdgpu_import_syncobj
;
1977 ws
->base
.export_syncobj_to_sync_file
= radv_amdgpu_export_syncobj_to_sync_file
;
1978 ws
->base
.import_syncobj_from_sync_file
= radv_amdgpu_import_syncobj_from_sync_file
;
1979 ws
->base
.fence_wait
= radv_amdgpu_fence_wait
;
1980 ws
->base
.fences_wait
= radv_amdgpu_fences_wait
;