2 * Copyright © 2008 Jérôme Glisse
3 * Copyright © 2010 Marek Olšák <maraeo@gmail.com>
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
16 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
18 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 * The above copyright notice and this permission notice (including the
24 * next paragraph) shall be included in all copies or substantial portions
29 * Marek Olšák <maraeo@gmail.com>
31 * Based on work from libdrm_radeon by:
32 * Aapo Tahkola <aet@rasterburn.org>
33 * Nicolai Haehnle <prefect_@gmx.net>
34 * Jérôme Glisse <glisse@freedesktop.org>
38 This file replaces libdrm's radeon_cs_gem with our own implemention.
39 It's optimized specifically for Radeon DRM.
40 Reloc writes and space checking are faster and simpler than their
41 counterparts in libdrm (the time complexity of all the functions
42 is O(1) in nearly all scenarios, thanks to hashing).
46 cs_add_reloc(cs, buf, read_domain, write_domain) adds a new relocation and
47 also adds the size of 'buf' to the used_gart and used_vram winsys variables
48 based on the domains, which are simply or'd for the accounting purposes.
49 The adding is skipped if the reloc is already present in the list, but it
50 accounts any newly-referenced domains.
52 cs_validate is then called, which just checks:
53 used_vram/gart < vram/gart_size * 0.8
54 The 0.8 number allows for some memory fragmentation. If the validation
55 fails, the pipe driver flushes CS and tries do the validation again,
56 i.e. it validates only that one operation. If it fails again, it drops
57 the operation on the floor and prints some nasty message to stderr.
58 (done in the pipe driver)
60 cs_write_reloc(cs, buf) just writes a reloc that has been added using
61 cs_add_reloc. The read_domain and write_domain parameters have been removed,
62 because we already specify them in cs_add_reloc.
65 #include "radeon_drm_cs.h"
67 #include "util/u_memory.h"
68 #include "os/os_time.h"
76 #define RELOC_DWORDS (sizeof(struct drm_radeon_cs_reloc) / sizeof(uint32_t))
78 static boolean
radeon_init_cs_context(struct radeon_cs_context
*csc
,
79 struct radeon_drm_winsys
*ws
)
83 csc
->relocs_bo
= (struct radeon_bo
**)
84 CALLOC(1, csc
->nrelocs
* sizeof(struct radeon_bo
*));
85 if (!csc
->relocs_bo
) {
89 csc
->relocs
= (struct drm_radeon_cs_reloc
*)
90 CALLOC(1, csc
->nrelocs
* sizeof(struct drm_radeon_cs_reloc
));
96 csc
->chunks
[0].chunk_id
= RADEON_CHUNK_ID_IB
;
97 csc
->chunks
[0].length_dw
= 0;
98 csc
->chunks
[0].chunk_data
= (uint64_t)(uintptr_t)csc
->buf
;
99 csc
->chunks
[1].chunk_id
= RADEON_CHUNK_ID_RELOCS
;
100 csc
->chunks
[1].length_dw
= 0;
101 csc
->chunks
[1].chunk_data
= (uint64_t)(uintptr_t)csc
->relocs
;
102 csc
->chunks
[2].chunk_id
= RADEON_CHUNK_ID_FLAGS
;
103 csc
->chunks
[2].length_dw
= 2;
104 csc
->chunks
[2].chunk_data
= (uint64_t)(uintptr_t)&csc
->flags
;
106 csc
->chunk_array
[0] = (uint64_t)(uintptr_t)&csc
->chunks
[0];
107 csc
->chunk_array
[1] = (uint64_t)(uintptr_t)&csc
->chunks
[1];
108 csc
->chunk_array
[2] = (uint64_t)(uintptr_t)&csc
->chunks
[2];
110 csc
->cs
.chunks
= (uint64_t)(uintptr_t)csc
->chunk_array
;
114 static void radeon_cs_context_cleanup(struct radeon_cs_context
*csc
)
118 for (i
= 0; i
< csc
->crelocs
; i
++) {
119 p_atomic_dec(&csc
->relocs_bo
[i
]->num_cs_references
);
120 radeon_bo_reference(&csc
->relocs_bo
[i
], NULL
);
124 csc
->validated_crelocs
= 0;
125 csc
->chunks
[0].length_dw
= 0;
126 csc
->chunks
[1].length_dw
= 0;
129 memset(csc
->is_handle_added
, 0, sizeof(csc
->is_handle_added
));
132 static void radeon_destroy_cs_context(struct radeon_cs_context
*csc
)
134 radeon_cs_context_cleanup(csc
);
135 FREE(csc
->relocs_bo
);
140 static struct radeon_winsys_cs
*
141 radeon_drm_cs_create(struct radeon_winsys
*rws
,
142 enum ring_type ring_type
,
143 void (*flush
)(void *ctx
, unsigned flags
),
145 struct radeon_winsys_cs_handle
*trace_buf
)
147 struct radeon_drm_winsys
*ws
= radeon_drm_winsys(rws
);
148 struct radeon_drm_cs
*cs
;
150 cs
= CALLOC_STRUCT(radeon_drm_cs
);
154 pipe_semaphore_init(&cs
->flush_completed
, 1);
157 cs
->flush_cs
= flush
;
158 cs
->flush_data
= flush_ctx
;
159 cs
->trace_buf
= (struct radeon_bo
*)trace_buf
;
161 if (!radeon_init_cs_context(&cs
->csc1
, cs
->ws
)) {
165 if (!radeon_init_cs_context(&cs
->csc2
, cs
->ws
)) {
166 radeon_destroy_cs_context(&cs
->csc1
);
171 /* Set the first command buffer as current. */
174 cs
->base
.buf
= cs
->csc
->buf
;
175 cs
->base
.ring_type
= ring_type
;
177 p_atomic_inc(&ws
->num_cs
);
181 #define OUT_CS(cs, value) (cs)->buf[(cs)->cdw++] = (value)
183 static INLINE
void update_reloc(struct drm_radeon_cs_reloc
*reloc
,
184 enum radeon_bo_domain rd
,
185 enum radeon_bo_domain wd
,
187 enum radeon_bo_domain
*added_domains
)
189 *added_domains
= (rd
| wd
) & ~(reloc
->read_domains
| reloc
->write_domain
);
191 reloc
->read_domains
|= rd
;
192 reloc
->write_domain
|= wd
;
193 reloc
->flags
= MAX2(reloc
->flags
, priority
);
196 int radeon_get_reloc(struct radeon_cs_context
*csc
, struct radeon_bo
*bo
,
197 struct drm_radeon_cs_reloc
**out_reloc
)
199 struct drm_radeon_cs_reloc
*reloc
= NULL
;
200 unsigned hash
= bo
->handle
& (sizeof(csc
->is_handle_added
)-1);
203 if (csc
->is_handle_added
[hash
]) {
204 i
= csc
->reloc_indices_hashlist
[hash
];
205 reloc
= &csc
->relocs
[i
];
207 if (reloc
->handle
!= bo
->handle
) {
208 /* Hash collision, look for the BO in the list of relocs linearly. */
209 for (i
= csc
->crelocs
- 1; i
>= 0; i
--) {
210 reloc
= &csc
->relocs
[i
];
211 if (reloc
->handle
== bo
->handle
) {
212 /* Put this reloc in the hash list.
213 * This will prevent additional hash collisions if there are
214 * several consecutive get_reloc calls for the same buffer.
216 * Example: Assuming buffers A,B,C collide in the hash list,
217 * the following sequence of relocs:
218 * AAAAAAAAAAABBBBBBBBBBBBBBCCCCCCCC
219 * will collide here: ^ and here: ^,
220 * meaning that we should get very few collisions in the end. */
221 csc
->reloc_indices_hashlist
[hash
] = i
;
232 static unsigned radeon_add_reloc(struct radeon_drm_cs
*cs
,
233 struct radeon_bo
*bo
,
234 enum radeon_bo_usage usage
,
235 enum radeon_bo_domain domains
,
237 enum radeon_bo_domain
*added_domains
)
239 struct radeon_cs_context
*csc
= cs
->csc
;
240 struct drm_radeon_cs_reloc
*reloc
;
241 unsigned hash
= bo
->handle
& (sizeof(csc
->is_handle_added
)-1);
242 enum radeon_bo_domain rd
= usage
& RADEON_USAGE_READ
? domains
: 0;
243 enum radeon_bo_domain wd
= usage
& RADEON_USAGE_WRITE
? domains
: 0;
246 priority
= MIN2(priority
, 15);
249 i
= radeon_get_reloc(csc
, bo
, &reloc
);
252 update_reloc(reloc
, rd
, wd
, priority
, added_domains
);
254 /* For async DMA, every add_reloc call must add a buffer to the list
255 * no matter how many duplicates there are. This is due to the fact
256 * the DMA CS checker doesn't use NOP packets for offset patching,
257 * but always uses the i-th buffer from the list to patch the i-th
258 * offset. If there are N offsets in a DMA CS, there must also be N
259 * buffers in the relocation list.
261 * This doesn't have to be done if virtual memory is enabled,
262 * because there is no offset patching with virtual memory.
264 if (cs
->base
.ring_type
!= RING_DMA
|| cs
->ws
->info
.r600_virtual_address
) {
269 /* New relocation, check if the backing array is large enough. */
270 if (csc
->crelocs
>= csc
->nrelocs
) {
274 size
= csc
->nrelocs
* sizeof(struct radeon_bo
*);
275 csc
->relocs_bo
= realloc(csc
->relocs_bo
, size
);
277 size
= csc
->nrelocs
* sizeof(struct drm_radeon_cs_reloc
);
278 csc
->relocs
= realloc(csc
->relocs
, size
);
280 csc
->chunks
[1].chunk_data
= (uint64_t)(uintptr_t)csc
->relocs
;
283 /* Initialize the new relocation. */
284 csc
->relocs_bo
[csc
->crelocs
] = NULL
;
285 radeon_bo_reference(&csc
->relocs_bo
[csc
->crelocs
], bo
);
286 p_atomic_inc(&bo
->num_cs_references
);
287 reloc
= &csc
->relocs
[csc
->crelocs
];
288 reloc
->handle
= bo
->handle
;
289 reloc
->read_domains
= rd
;
290 reloc
->write_domain
= wd
;
291 reloc
->flags
= priority
;
293 csc
->is_handle_added
[hash
] = TRUE
;
294 csc
->reloc_indices_hashlist
[hash
] = csc
->crelocs
;
296 csc
->chunks
[1].length_dw
+= RELOC_DWORDS
;
298 *added_domains
= rd
| wd
;
299 return csc
->crelocs
++;
302 static unsigned radeon_drm_cs_add_reloc(struct radeon_winsys_cs
*rcs
,
303 struct radeon_winsys_cs_handle
*buf
,
304 enum radeon_bo_usage usage
,
305 enum radeon_bo_domain domains
,
306 enum radeon_bo_priority priority
)
308 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
309 struct radeon_bo
*bo
= (struct radeon_bo
*)buf
;
310 enum radeon_bo_domain added_domains
;
311 unsigned index
= radeon_add_reloc(cs
, bo
, usage
, domains
, priority
, &added_domains
);
313 if (added_domains
& RADEON_DOMAIN_GTT
)
314 cs
->csc
->used_gart
+= bo
->base
.size
;
315 if (added_domains
& RADEON_DOMAIN_VRAM
)
316 cs
->csc
->used_vram
+= bo
->base
.size
;
321 static int radeon_drm_cs_get_reloc(struct radeon_winsys_cs
*rcs
,
322 struct radeon_winsys_cs_handle
*buf
)
324 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
326 return radeon_get_reloc(cs
->csc
, (struct radeon_bo
*)buf
, NULL
);
329 static boolean
radeon_drm_cs_validate(struct radeon_winsys_cs
*rcs
)
331 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
333 cs
->csc
->used_gart
< cs
->ws
->info
.gart_size
* 0.8 &&
334 cs
->csc
->used_vram
< cs
->ws
->info
.vram_size
* 0.8;
337 cs
->csc
->validated_crelocs
= cs
->csc
->crelocs
;
339 /* Remove lately-added relocations. The validation failed with them
340 * and the CS is about to be flushed because of that. Keep only
341 * the already-validated relocations. */
344 for (i
= cs
->csc
->validated_crelocs
; i
< cs
->csc
->crelocs
; i
++) {
345 p_atomic_dec(&cs
->csc
->relocs_bo
[i
]->num_cs_references
);
346 radeon_bo_reference(&cs
->csc
->relocs_bo
[i
], NULL
);
348 cs
->csc
->crelocs
= cs
->csc
->validated_crelocs
;
350 /* Flush if there are any relocs. Clean up otherwise. */
351 if (cs
->csc
->crelocs
) {
352 cs
->flush_cs(cs
->flush_data
, RADEON_FLUSH_ASYNC
);
354 radeon_cs_context_cleanup(cs
->csc
);
356 assert(cs
->base
.cdw
== 0);
357 if (cs
->base
.cdw
!= 0) {
358 fprintf(stderr
, "radeon: Unexpected error in %s.\n", __func__
);
365 static boolean
radeon_drm_cs_memory_below_limit(struct radeon_winsys_cs
*rcs
, uint64_t vram
, uint64_t gtt
)
367 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
369 (cs
->csc
->used_gart
+ gtt
) < cs
->ws
->info
.gart_size
* 0.7 &&
370 (cs
->csc
->used_vram
+ vram
) < cs
->ws
->info
.vram_size
* 0.7;
375 void radeon_drm_cs_emit_ioctl_oneshot(struct radeon_drm_cs
*cs
, struct radeon_cs_context
*csc
)
379 if (drmCommandWriteRead(csc
->fd
, DRM_RADEON_CS
,
380 &csc
->cs
, sizeof(struct drm_radeon_cs
))) {
381 if (debug_get_bool_option("RADEON_DUMP_CS", FALSE
)) {
384 fprintf(stderr
, "radeon: The kernel rejected CS, dumping...\n");
385 for (i
= 0; i
< csc
->chunks
[0].length_dw
; i
++) {
386 fprintf(stderr
, "0x%08X\n", csc
->buf
[i
]);
389 fprintf(stderr
, "radeon: The kernel rejected CS, "
390 "see dmesg for more information.\n");
395 radeon_dump_cs_on_lockup(cs
, csc
);
398 for (i
= 0; i
< csc
->crelocs
; i
++)
399 p_atomic_dec(&csc
->relocs_bo
[i
]->num_active_ioctls
);
401 radeon_cs_context_cleanup(csc
);
405 * Make sure previous submission of this cs are completed
407 void radeon_drm_cs_sync_flush(struct radeon_winsys_cs
*rcs
)
409 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
411 /* Wait for any pending ioctl to complete. */
412 if (cs
->ws
->thread
) {
413 pipe_semaphore_wait(&cs
->flush_completed
);
414 pipe_semaphore_signal(&cs
->flush_completed
);
418 DEBUG_GET_ONCE_BOOL_OPTION(noop
, "RADEON_NOOP", FALSE
)
420 static void radeon_drm_cs_flush(struct radeon_winsys_cs
*rcs
, unsigned flags
, uint32_t cs_trace_id
)
422 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
423 struct radeon_cs_context
*tmp
;
425 switch (cs
->base
.ring_type
) {
427 /* pad DMA ring to 8 DWs */
428 if (cs
->ws
->info
.chip_class
<= SI
) {
430 OUT_CS(&cs
->base
, 0xf0000000); /* NOP packet */
433 OUT_CS(&cs
->base
, 0x00000000); /* NOP packet */
437 /* pad DMA ring to 8 DWs to meet CP fetch alignment requirements
438 * r6xx, requires at least 4 dw alignment to avoid a hw bug.
440 if (cs
->ws
->info
.chip_class
<= SI
) {
442 OUT_CS(&cs
->base
, 0x80000000); /* type2 nop packet */
445 OUT_CS(&cs
->base
, 0xffff1000); /* type3 nop packet */
449 while (rcs
->cdw
& 15)
450 OUT_CS(&cs
->base
, 0x80000000); /* type2 nop packet */
456 if (rcs
->cdw
> RADEON_MAX_CMDBUF_DWORDS
) {
457 fprintf(stderr
, "radeon: command stream overflowed\n");
460 radeon_drm_cs_sync_flush(rcs
);
462 /* Flip command streams. */
467 cs
->cst
->cs_trace_id
= cs_trace_id
;
469 /* If the CS is not empty or overflowed, emit it in a separate thread. */
470 if (cs
->base
.cdw
&& cs
->base
.cdw
<= RADEON_MAX_CMDBUF_DWORDS
&& !debug_get_option_noop()) {
471 unsigned i
, crelocs
= cs
->cst
->crelocs
;
473 cs
->cst
->chunks
[0].length_dw
= cs
->base
.cdw
;
475 for (i
= 0; i
< crelocs
; i
++) {
476 /* Update the number of active asynchronous CS ioctls for the buffer. */
477 p_atomic_inc(&cs
->cst
->relocs_bo
[i
]->num_active_ioctls
);
480 switch (cs
->base
.ring_type
) {
482 cs
->cst
->flags
[0] = 0;
483 cs
->cst
->flags
[1] = RADEON_CS_RING_DMA
;
484 cs
->cst
->cs
.num_chunks
= 3;
485 if (cs
->ws
->info
.r600_virtual_address
) {
486 cs
->cst
->flags
[0] |= RADEON_CS_USE_VM
;
491 cs
->cst
->flags
[0] = 0;
492 cs
->cst
->flags
[1] = RADEON_CS_RING_UVD
;
493 cs
->cst
->cs
.num_chunks
= 3;
497 cs
->cst
->flags
[0] = 0;
498 cs
->cst
->flags
[1] = RADEON_CS_RING_VCE
;
499 cs
->cst
->cs
.num_chunks
= 3;
504 cs
->cst
->flags
[0] = 0;
505 cs
->cst
->flags
[1] = RADEON_CS_RING_GFX
;
506 cs
->cst
->cs
.num_chunks
= 2;
507 if (flags
& RADEON_FLUSH_KEEP_TILING_FLAGS
) {
508 cs
->cst
->flags
[0] |= RADEON_CS_KEEP_TILING_FLAGS
;
509 cs
->cst
->cs
.num_chunks
= 3;
511 if (cs
->ws
->info
.r600_virtual_address
) {
512 cs
->cst
->flags
[0] |= RADEON_CS_USE_VM
;
513 cs
->cst
->cs
.num_chunks
= 3;
515 if (flags
& RADEON_FLUSH_END_OF_FRAME
) {
516 cs
->cst
->flags
[0] |= RADEON_CS_END_OF_FRAME
;
517 cs
->cst
->cs
.num_chunks
= 3;
519 if (flags
& RADEON_FLUSH_COMPUTE
) {
520 cs
->cst
->flags
[1] = RADEON_CS_RING_COMPUTE
;
521 cs
->cst
->cs
.num_chunks
= 3;
526 if (cs
->ws
->thread
) {
527 pipe_semaphore_wait(&cs
->flush_completed
);
528 radeon_drm_ws_queue_cs(cs
->ws
, cs
);
529 if (!(flags
& RADEON_FLUSH_ASYNC
))
530 radeon_drm_cs_sync_flush(rcs
);
532 radeon_drm_cs_emit_ioctl_oneshot(cs
, cs
->cst
);
535 radeon_cs_context_cleanup(cs
->cst
);
538 /* Prepare a new CS. */
539 cs
->base
.buf
= cs
->csc
->buf
;
542 cs
->ws
->num_cs_flushes
++;
545 static void radeon_drm_cs_destroy(struct radeon_winsys_cs
*rcs
)
547 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
549 radeon_drm_cs_sync_flush(rcs
);
550 pipe_semaphore_destroy(&cs
->flush_completed
);
551 radeon_cs_context_cleanup(&cs
->csc1
);
552 radeon_cs_context_cleanup(&cs
->csc2
);
553 p_atomic_dec(&cs
->ws
->num_cs
);
554 radeon_destroy_cs_context(&cs
->csc1
);
555 radeon_destroy_cs_context(&cs
->csc2
);
559 static boolean
radeon_bo_is_referenced(struct radeon_winsys_cs
*rcs
,
560 struct radeon_winsys_cs_handle
*_buf
,
561 enum radeon_bo_usage usage
)
563 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
564 struct radeon_bo
*bo
= (struct radeon_bo
*)_buf
;
567 if (!bo
->num_cs_references
)
570 index
= radeon_get_reloc(cs
->csc
, bo
, NULL
);
574 if ((usage
& RADEON_USAGE_WRITE
) && cs
->csc
->relocs
[index
].write_domain
)
576 if ((usage
& RADEON_USAGE_READ
) && cs
->csc
->relocs
[index
].read_domains
)
584 static struct pipe_fence_handle
*
585 radeon_cs_create_fence(struct radeon_winsys_cs
*rcs
)
587 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
588 struct pb_buffer
*fence
;
590 /* Create a fence, which is a dummy BO. */
591 fence
= cs
->ws
->base
.buffer_create(&cs
->ws
->base
, 1, 1, TRUE
,
593 /* Add the fence as a dummy relocation. */
594 cs
->ws
->base
.cs_add_reloc(rcs
, cs
->ws
->base
.buffer_get_cs_handle(fence
),
595 RADEON_USAGE_READWRITE
, RADEON_DOMAIN_GTT
,
597 return (struct pipe_fence_handle
*)fence
;
600 static bool radeon_fence_wait(struct radeon_winsys
*ws
,
601 struct pipe_fence_handle
*fence
,
604 struct pb_buffer
*rfence
= (struct pb_buffer
*)fence
;
607 return !ws
->buffer_is_busy(rfence
, RADEON_USAGE_READWRITE
);
609 if (timeout
!= PIPE_TIMEOUT_INFINITE
) {
610 int64_t start_time
= os_time_get();
612 /* Convert to microseconds. */
615 /* Wait in a loop. */
616 while (ws
->buffer_is_busy(rfence
, RADEON_USAGE_READWRITE
)) {
617 if (os_time_get() - start_time
>= timeout
) {
625 ws
->buffer_wait(rfence
, RADEON_USAGE_READWRITE
);
629 static void radeon_fence_reference(struct pipe_fence_handle
**dst
,
630 struct pipe_fence_handle
*src
)
632 pb_reference((struct pb_buffer
**)dst
, (struct pb_buffer
*)src
);
635 void radeon_drm_cs_init_functions(struct radeon_drm_winsys
*ws
)
637 ws
->base
.cs_create
= radeon_drm_cs_create
;
638 ws
->base
.cs_destroy
= radeon_drm_cs_destroy
;
639 ws
->base
.cs_add_reloc
= radeon_drm_cs_add_reloc
;
640 ws
->base
.cs_get_reloc
= radeon_drm_cs_get_reloc
;
641 ws
->base
.cs_validate
= radeon_drm_cs_validate
;
642 ws
->base
.cs_memory_below_limit
= radeon_drm_cs_memory_below_limit
;
643 ws
->base
.cs_flush
= radeon_drm_cs_flush
;
644 ws
->base
.cs_is_buffer_referenced
= radeon_bo_is_referenced
;
645 ws
->base
.cs_sync_flush
= radeon_drm_cs_sync_flush
;
646 ws
->base
.cs_create_fence
= radeon_cs_create_fence
;
647 ws
->base
.fence_wait
= radeon_fence_wait
;
648 ws
->base
.fence_reference
= radeon_fence_reference
;