2 * Copyright © 2008 Jérôme Glisse
3 * Copyright © 2010 Marek Olšák <maraeo@gmail.com>
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
16 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
18 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 * The above copyright notice and this permission notice (including the
24 * next paragraph) shall be included in all copies or substantial portions
29 This file replaces libdrm's radeon_cs_gem with our own implemention.
30 It's optimized specifically for Radeon DRM.
31 Adding buffers and space checking are faster and simpler than their
32 counterparts in libdrm (the time complexity of all the functions
33 is O(1) in nearly all scenarios, thanks to hashing).
37 cs_add_buffer(cs, buf, read_domain, write_domain) adds a new relocation and
38 also adds the size of 'buf' to the used_gart and used_vram winsys variables
39 based on the domains, which are simply or'd for the accounting purposes.
40 The adding is skipped if the reloc is already present in the list, but it
41 accounts any newly-referenced domains.
43 cs_validate is then called, which just checks:
44 used_vram/gart < vram/gart_size * 0.8
45 The 0.8 number allows for some memory fragmentation. If the validation
46 fails, the pipe driver flushes CS and tries do the validation again,
47 i.e. it validates only that one operation. If it fails again, it drops
48 the operation on the floor and prints some nasty message to stderr.
49 (done in the pipe driver)
51 cs_write_reloc(cs, buf) just writes a reloc that has been added using
52 cs_add_buffer. The read_domain and write_domain parameters have been removed,
53 because we already specify them in cs_add_buffer.
56 #include "radeon_drm_cs.h"
58 #include "util/u_memory.h"
59 #include "util/os_time.h"
67 #define RELOC_DWORDS (sizeof(struct drm_radeon_cs_reloc) / sizeof(uint32_t))
69 static struct pipe_fence_handle
*
70 radeon_cs_create_fence(struct radeon_cmdbuf
*rcs
);
71 static void radeon_fence_reference(struct pipe_fence_handle
**dst
,
72 struct pipe_fence_handle
*src
);
74 static struct radeon_winsys_ctx
*radeon_drm_ctx_create(struct radeon_winsys
*ws
)
76 /* No context support here. Just return the winsys pointer
77 * as the "context". */
78 return (struct radeon_winsys_ctx
*)ws
;
81 static void radeon_drm_ctx_destroy(struct radeon_winsys_ctx
*ctx
)
83 /* No context support here. */
86 static bool radeon_init_cs_context(struct radeon_cs_context
*csc
,
87 struct radeon_drm_winsys
*ws
)
93 csc
->chunks
[0].chunk_id
= RADEON_CHUNK_ID_IB
;
94 csc
->chunks
[0].length_dw
= 0;
95 csc
->chunks
[0].chunk_data
= (uint64_t)(uintptr_t)csc
->buf
;
96 csc
->chunks
[1].chunk_id
= RADEON_CHUNK_ID_RELOCS
;
97 csc
->chunks
[1].length_dw
= 0;
98 csc
->chunks
[1].chunk_data
= (uint64_t)(uintptr_t)csc
->relocs
;
99 csc
->chunks
[2].chunk_id
= RADEON_CHUNK_ID_FLAGS
;
100 csc
->chunks
[2].length_dw
= 2;
101 csc
->chunks
[2].chunk_data
= (uint64_t)(uintptr_t)&csc
->flags
;
103 csc
->chunk_array
[0] = (uint64_t)(uintptr_t)&csc
->chunks
[0];
104 csc
->chunk_array
[1] = (uint64_t)(uintptr_t)&csc
->chunks
[1];
105 csc
->chunk_array
[2] = (uint64_t)(uintptr_t)&csc
->chunks
[2];
107 csc
->cs
.chunks
= (uint64_t)(uintptr_t)csc
->chunk_array
;
109 for (i
= 0; i
< ARRAY_SIZE(csc
->reloc_indices_hashlist
); i
++) {
110 csc
->reloc_indices_hashlist
[i
] = -1;
115 static void radeon_cs_context_cleanup(struct radeon_cs_context
*csc
)
119 for (i
= 0; i
< csc
->num_relocs
; i
++) {
120 p_atomic_dec(&csc
->relocs_bo
[i
].bo
->num_cs_references
);
121 radeon_bo_reference(&csc
->relocs_bo
[i
].bo
, NULL
);
123 for (i
= 0; i
< csc
->num_slab_buffers
; ++i
) {
124 p_atomic_dec(&csc
->slab_buffers
[i
].bo
->num_cs_references
);
125 radeon_bo_reference(&csc
->slab_buffers
[i
].bo
, NULL
);
129 csc
->num_validated_relocs
= 0;
130 csc
->num_slab_buffers
= 0;
131 csc
->chunks
[0].length_dw
= 0;
132 csc
->chunks
[1].length_dw
= 0;
134 for (i
= 0; i
< ARRAY_SIZE(csc
->reloc_indices_hashlist
); i
++) {
135 csc
->reloc_indices_hashlist
[i
] = -1;
139 static void radeon_destroy_cs_context(struct radeon_cs_context
*csc
)
141 radeon_cs_context_cleanup(csc
);
142 FREE(csc
->slab_buffers
);
143 FREE(csc
->relocs_bo
);
148 static struct radeon_cmdbuf
*
149 radeon_drm_cs_create(struct radeon_winsys_ctx
*ctx
,
150 enum ring_type ring_type
,
151 void (*flush
)(void *ctx
, unsigned flags
,
152 struct pipe_fence_handle
**fence
),
154 bool stop_exec_on_failure
)
156 struct radeon_drm_winsys
*ws
= (struct radeon_drm_winsys
*)ctx
;
157 struct radeon_drm_cs
*cs
;
159 cs
= CALLOC_STRUCT(radeon_drm_cs
);
163 util_queue_fence_init(&cs
->flush_completed
);
166 cs
->flush_cs
= flush
;
167 cs
->flush_data
= flush_ctx
;
169 if (!radeon_init_cs_context(&cs
->csc1
, cs
->ws
)) {
173 if (!radeon_init_cs_context(&cs
->csc2
, cs
->ws
)) {
174 radeon_destroy_cs_context(&cs
->csc1
);
179 /* Set the first command buffer as current. */
182 cs
->base
.current
.buf
= cs
->csc
->buf
;
183 cs
->base
.current
.max_dw
= ARRAY_SIZE(cs
->csc
->buf
);
184 cs
->ring_type
= ring_type
;
186 p_atomic_inc(&ws
->num_cs
);
190 int radeon_lookup_buffer(struct radeon_cs_context
*csc
, struct radeon_bo
*bo
)
192 unsigned hash
= bo
->hash
& (ARRAY_SIZE(csc
->reloc_indices_hashlist
)-1);
193 struct radeon_bo_item
*buffers
;
194 unsigned num_buffers
;
195 int i
= csc
->reloc_indices_hashlist
[hash
];
198 buffers
= csc
->relocs_bo
;
199 num_buffers
= csc
->num_relocs
;
201 buffers
= csc
->slab_buffers
;
202 num_buffers
= csc
->num_slab_buffers
;
205 /* not found or found */
206 if (i
== -1 || (i
< num_buffers
&& buffers
[i
].bo
== bo
))
209 /* Hash collision, look for the BO in the list of relocs linearly. */
210 for (i
= num_buffers
- 1; i
>= 0; i
--) {
211 if (buffers
[i
].bo
== bo
) {
212 /* Put this reloc in the hash list.
213 * This will prevent additional hash collisions if there are
214 * several consecutive lookup_buffer calls for the same buffer.
216 * Example: Assuming buffers A,B,C collide in the hash list,
217 * the following sequence of relocs:
218 * AAAAAAAAAAABBBBBBBBBBBBBBCCCCCCCC
219 * will collide here: ^ and here: ^,
220 * meaning that we should get very few collisions in the end. */
221 csc
->reloc_indices_hashlist
[hash
] = i
;
228 static unsigned radeon_lookup_or_add_real_buffer(struct radeon_drm_cs
*cs
,
229 struct radeon_bo
*bo
)
231 struct radeon_cs_context
*csc
= cs
->csc
;
232 struct drm_radeon_cs_reloc
*reloc
;
233 unsigned hash
= bo
->hash
& (ARRAY_SIZE(csc
->reloc_indices_hashlist
)-1);
236 i
= radeon_lookup_buffer(csc
, bo
);
239 /* For async DMA, every add_buffer call must add a buffer to the list
240 * no matter how many duplicates there are. This is due to the fact
241 * the DMA CS checker doesn't use NOP packets for offset patching,
242 * but always uses the i-th buffer from the list to patch the i-th
243 * offset. If there are N offsets in a DMA CS, there must also be N
244 * buffers in the relocation list.
246 * This doesn't have to be done if virtual memory is enabled,
247 * because there is no offset patching with virtual memory.
249 if (cs
->ring_type
!= RING_DMA
|| cs
->ws
->info
.r600_has_virtual_memory
) {
254 /* New relocation, check if the backing array is large enough. */
255 if (csc
->num_relocs
>= csc
->max_relocs
) {
257 csc
->max_relocs
= MAX2(csc
->max_relocs
+ 16, (unsigned)(csc
->max_relocs
* 1.3));
259 size
= csc
->max_relocs
* sizeof(csc
->relocs_bo
[0]);
260 csc
->relocs_bo
= realloc(csc
->relocs_bo
, size
);
262 size
= csc
->max_relocs
* sizeof(struct drm_radeon_cs_reloc
);
263 csc
->relocs
= realloc(csc
->relocs
, size
);
265 csc
->chunks
[1].chunk_data
= (uint64_t)(uintptr_t)csc
->relocs
;
268 /* Initialize the new relocation. */
269 csc
->relocs_bo
[csc
->num_relocs
].bo
= NULL
;
270 csc
->relocs_bo
[csc
->num_relocs
].u
.real
.priority_usage
= 0;
271 radeon_bo_reference(&csc
->relocs_bo
[csc
->num_relocs
].bo
, bo
);
272 p_atomic_inc(&bo
->num_cs_references
);
273 reloc
= &csc
->relocs
[csc
->num_relocs
];
274 reloc
->handle
= bo
->handle
;
275 reloc
->read_domains
= 0;
276 reloc
->write_domain
= 0;
279 csc
->reloc_indices_hashlist
[hash
] = csc
->num_relocs
;
281 csc
->chunks
[1].length_dw
+= RELOC_DWORDS
;
283 return csc
->num_relocs
++;
286 static int radeon_lookup_or_add_slab_buffer(struct radeon_drm_cs
*cs
,
287 struct radeon_bo
*bo
)
289 struct radeon_cs_context
*csc
= cs
->csc
;
291 struct radeon_bo_item
*item
;
295 idx
= radeon_lookup_buffer(csc
, bo
);
299 real_idx
= radeon_lookup_or_add_real_buffer(cs
, bo
->u
.slab
.real
);
301 /* Check if the backing array is large enough. */
302 if (csc
->num_slab_buffers
>= csc
->max_slab_buffers
) {
303 unsigned new_max
= MAX2(csc
->max_slab_buffers
+ 16,
304 (unsigned)(csc
->max_slab_buffers
* 1.3));
305 struct radeon_bo_item
*new_buffers
=
306 REALLOC(csc
->slab_buffers
,
307 csc
->max_slab_buffers
* sizeof(*new_buffers
),
308 new_max
* sizeof(*new_buffers
));
310 fprintf(stderr
, "radeon_lookup_or_add_slab_buffer: allocation failure\n");
314 csc
->max_slab_buffers
= new_max
;
315 csc
->slab_buffers
= new_buffers
;
318 /* Initialize the new relocation. */
319 idx
= csc
->num_slab_buffers
++;
320 item
= &csc
->slab_buffers
[idx
];
323 item
->u
.slab
.real_idx
= real_idx
;
324 radeon_bo_reference(&item
->bo
, bo
);
325 p_atomic_inc(&bo
->num_cs_references
);
327 hash
= bo
->hash
& (ARRAY_SIZE(csc
->reloc_indices_hashlist
)-1);
328 csc
->reloc_indices_hashlist
[hash
] = idx
;
333 static unsigned radeon_drm_cs_add_buffer(struct radeon_cmdbuf
*rcs
,
334 struct pb_buffer
*buf
,
335 enum radeon_bo_usage usage
,
336 enum radeon_bo_domain domains
,
337 enum radeon_bo_priority priority
)
339 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
340 struct radeon_bo
*bo
= (struct radeon_bo
*)buf
;
341 enum radeon_bo_domain added_domains
;
343 /* If VRAM is just stolen system memory, allow both VRAM and
344 * GTT, whichever has free space. If a buffer is evicted from
345 * VRAM to GTT, it will stay there.
347 if (!cs
->ws
->info
.has_dedicated_vram
)
348 domains
|= RADEON_DOMAIN_GTT
;
350 enum radeon_bo_domain rd
= usage
& RADEON_USAGE_READ
? domains
: 0;
351 enum radeon_bo_domain wd
= usage
& RADEON_USAGE_WRITE
? domains
: 0;
352 struct drm_radeon_cs_reloc
*reloc
;
356 index
= radeon_lookup_or_add_slab_buffer(cs
, bo
);
360 index
= cs
->csc
->slab_buffers
[index
].u
.slab
.real_idx
;
362 index
= radeon_lookup_or_add_real_buffer(cs
, bo
);
365 reloc
= &cs
->csc
->relocs
[index
];
366 added_domains
= (rd
| wd
) & ~(reloc
->read_domains
| reloc
->write_domain
);
367 reloc
->read_domains
|= rd
;
368 reloc
->write_domain
|= wd
;
369 reloc
->flags
= MAX2(reloc
->flags
, priority
);
370 cs
->csc
->relocs_bo
[index
].u
.real
.priority_usage
|= 1u << priority
;
372 if (added_domains
& RADEON_DOMAIN_VRAM
)
373 cs
->base
.used_vram
+= bo
->base
.size
;
374 else if (added_domains
& RADEON_DOMAIN_GTT
)
375 cs
->base
.used_gart
+= bo
->base
.size
;
380 static int radeon_drm_cs_lookup_buffer(struct radeon_cmdbuf
*rcs
,
381 struct pb_buffer
*buf
)
383 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
385 return radeon_lookup_buffer(cs
->csc
, (struct radeon_bo
*)buf
);
388 static bool radeon_drm_cs_validate(struct radeon_cmdbuf
*rcs
)
390 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
392 cs
->base
.used_gart
< cs
->ws
->info
.gart_size
* 0.8 &&
393 cs
->base
.used_vram
< cs
->ws
->info
.vram_size
* 0.8;
396 cs
->csc
->num_validated_relocs
= cs
->csc
->num_relocs
;
398 /* Remove lately-added buffers. The validation failed with them
399 * and the CS is about to be flushed because of that. Keep only
400 * the already-validated buffers. */
403 for (i
= cs
->csc
->num_validated_relocs
; i
< cs
->csc
->num_relocs
; i
++) {
404 p_atomic_dec(&cs
->csc
->relocs_bo
[i
].bo
->num_cs_references
);
405 radeon_bo_reference(&cs
->csc
->relocs_bo
[i
].bo
, NULL
);
407 cs
->csc
->num_relocs
= cs
->csc
->num_validated_relocs
;
409 /* Flush if there are any relocs. Clean up otherwise. */
410 if (cs
->csc
->num_relocs
) {
411 cs
->flush_cs(cs
->flush_data
,
412 RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW
, NULL
);
414 radeon_cs_context_cleanup(cs
->csc
);
415 cs
->base
.used_vram
= 0;
416 cs
->base
.used_gart
= 0;
418 assert(cs
->base
.current
.cdw
== 0);
419 if (cs
->base
.current
.cdw
!= 0) {
420 fprintf(stderr
, "radeon: Unexpected error in %s.\n", __func__
);
427 static bool radeon_drm_cs_check_space(struct radeon_cmdbuf
*rcs
, unsigned dw
)
429 assert(rcs
->current
.cdw
<= rcs
->current
.max_dw
);
430 return rcs
->current
.max_dw
- rcs
->current
.cdw
>= dw
;
433 static unsigned radeon_drm_cs_get_buffer_list(struct radeon_cmdbuf
*rcs
,
434 struct radeon_bo_list_item
*list
)
436 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
440 for (i
= 0; i
< cs
->csc
->num_relocs
; i
++) {
441 list
[i
].bo_size
= cs
->csc
->relocs_bo
[i
].bo
->base
.size
;
442 list
[i
].vm_address
= cs
->csc
->relocs_bo
[i
].bo
->va
;
443 list
[i
].priority_usage
= cs
->csc
->relocs_bo
[i
].u
.real
.priority_usage
;
446 return cs
->csc
->num_relocs
;
449 void radeon_drm_cs_emit_ioctl_oneshot(void *job
, int thread_index
)
451 struct radeon_cs_context
*csc
= ((struct radeon_drm_cs
*)job
)->cst
;
455 r
= drmCommandWriteRead(csc
->fd
, DRM_RADEON_CS
,
456 &csc
->cs
, sizeof(struct drm_radeon_cs
));
459 fprintf(stderr
, "radeon: Not enough memory for command submission.\n");
460 else if (debug_get_bool_option("RADEON_DUMP_CS", false)) {
463 fprintf(stderr
, "radeon: The kernel rejected CS, dumping...\n");
464 for (i
= 0; i
< csc
->chunks
[0].length_dw
; i
++) {
465 fprintf(stderr
, "0x%08X\n", csc
->buf
[i
]);
468 fprintf(stderr
, "radeon: The kernel rejected CS, "
469 "see dmesg for more information (%i).\n", r
);
473 for (i
= 0; i
< csc
->num_relocs
; i
++)
474 p_atomic_dec(&csc
->relocs_bo
[i
].bo
->num_active_ioctls
);
475 for (i
= 0; i
< csc
->num_slab_buffers
; i
++)
476 p_atomic_dec(&csc
->slab_buffers
[i
].bo
->num_active_ioctls
);
478 radeon_cs_context_cleanup(csc
);
482 * Make sure previous submission of this cs are completed
484 void radeon_drm_cs_sync_flush(struct radeon_cmdbuf
*rcs
)
486 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
488 /* Wait for any pending ioctl of this CS to complete. */
489 if (util_queue_is_initialized(&cs
->ws
->cs_queue
))
490 util_queue_fence_wait(&cs
->flush_completed
);
493 /* Add the given fence to a slab buffer fence list.
495 * There is a potential race condition when bo participates in submissions on
496 * two or more threads simultaneously. Since we do not know which of the
497 * submissions will be sent to the GPU first, we have to keep the fences
498 * of all submissions.
500 * However, fences that belong to submissions that have already returned from
501 * their respective ioctl do not have to be kept, because we know that they
502 * will signal earlier.
504 static void radeon_bo_slab_fence(struct radeon_bo
*bo
, struct radeon_bo
*fence
)
508 assert(fence
->num_cs_references
);
510 /* Cleanup older fences */
512 for (unsigned src
= 0; src
< bo
->u
.slab
.num_fences
; ++src
) {
513 if (bo
->u
.slab
.fences
[src
]->num_cs_references
) {
514 bo
->u
.slab
.fences
[dst
] = bo
->u
.slab
.fences
[src
];
517 radeon_bo_reference(&bo
->u
.slab
.fences
[src
], NULL
);
520 bo
->u
.slab
.num_fences
= dst
;
522 /* Check available space for the new fence */
523 if (bo
->u
.slab
.num_fences
>= bo
->u
.slab
.max_fences
) {
524 unsigned new_max_fences
= bo
->u
.slab
.max_fences
+ 1;
525 struct radeon_bo
**new_fences
= REALLOC(bo
->u
.slab
.fences
,
526 bo
->u
.slab
.max_fences
* sizeof(*new_fences
),
527 new_max_fences
* sizeof(*new_fences
));
529 fprintf(stderr
, "radeon_bo_slab_fence: allocation failure, dropping fence\n");
533 bo
->u
.slab
.fences
= new_fences
;
534 bo
->u
.slab
.max_fences
= new_max_fences
;
537 /* Add the new fence */
538 bo
->u
.slab
.fences
[bo
->u
.slab
.num_fences
] = NULL
;
539 radeon_bo_reference(&bo
->u
.slab
.fences
[bo
->u
.slab
.num_fences
], fence
);
540 bo
->u
.slab
.num_fences
++;
543 DEBUG_GET_ONCE_BOOL_OPTION(noop
, "RADEON_NOOP", false)
545 static int radeon_drm_cs_flush(struct radeon_cmdbuf
*rcs
,
547 struct pipe_fence_handle
**pfence
)
549 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
550 struct radeon_cs_context
*tmp
;
552 switch (cs
->ring_type
) {
554 /* pad DMA ring to 8 DWs */
555 if (cs
->ws
->info
.chip_class
<= GFX6
) {
556 while (rcs
->current
.cdw
& 7)
557 radeon_emit(&cs
->base
, 0xf0000000); /* NOP packet */
559 while (rcs
->current
.cdw
& 7)
560 radeon_emit(&cs
->base
, 0x00000000); /* NOP packet */
564 /* pad GFX ring to 8 DWs to meet CP fetch alignment requirements
565 * r6xx, requires at least 4 dw alignment to avoid a hw bug.
567 if (cs
->ws
->info
.gfx_ib_pad_with_type2
) {
568 while (rcs
->current
.cdw
& 7)
569 radeon_emit(&cs
->base
, 0x80000000); /* type2 nop packet */
571 while (rcs
->current
.cdw
& 7)
572 radeon_emit(&cs
->base
, 0xffff1000); /* type3 nop packet */
576 while (rcs
->current
.cdw
& 15)
577 radeon_emit(&cs
->base
, 0x80000000); /* type2 nop packet */
583 if (rcs
->current
.cdw
> rcs
->current
.max_dw
) {
584 fprintf(stderr
, "radeon: command stream overflowed\n");
587 if (pfence
|| cs
->csc
->num_slab_buffers
) {
588 struct pipe_fence_handle
*fence
;
590 if (cs
->next_fence
) {
591 fence
= cs
->next_fence
;
592 cs
->next_fence
= NULL
;
594 fence
= radeon_cs_create_fence(rcs
);
599 radeon_fence_reference(pfence
, fence
);
601 mtx_lock(&cs
->ws
->bo_fence_lock
);
602 for (unsigned i
= 0; i
< cs
->csc
->num_slab_buffers
; ++i
) {
603 struct radeon_bo
*bo
= cs
->csc
->slab_buffers
[i
].bo
;
604 p_atomic_inc(&bo
->num_active_ioctls
);
605 radeon_bo_slab_fence(bo
, (struct radeon_bo
*)fence
);
607 mtx_unlock(&cs
->ws
->bo_fence_lock
);
609 radeon_fence_reference(&fence
, NULL
);
612 radeon_fence_reference(&cs
->next_fence
, NULL
);
615 radeon_drm_cs_sync_flush(rcs
);
617 /* Swap command streams. */
622 /* If the CS is not empty or overflowed, emit it in a separate thread. */
623 if (cs
->base
.current
.cdw
&& cs
->base
.current
.cdw
<= cs
->base
.current
.max_dw
&& !debug_get_option_noop()) {
624 unsigned i
, num_relocs
;
626 num_relocs
= cs
->cst
->num_relocs
;
628 cs
->cst
->chunks
[0].length_dw
= cs
->base
.current
.cdw
;
630 for (i
= 0; i
< num_relocs
; i
++) {
631 /* Update the number of active asynchronous CS ioctls for the buffer. */
632 p_atomic_inc(&cs
->cst
->relocs_bo
[i
].bo
->num_active_ioctls
);
635 switch (cs
->ring_type
) {
637 cs
->cst
->flags
[0] = 0;
638 cs
->cst
->flags
[1] = RADEON_CS_RING_DMA
;
639 cs
->cst
->cs
.num_chunks
= 3;
640 if (cs
->ws
->info
.r600_has_virtual_memory
) {
641 cs
->cst
->flags
[0] |= RADEON_CS_USE_VM
;
646 cs
->cst
->flags
[0] = 0;
647 cs
->cst
->flags
[1] = RADEON_CS_RING_UVD
;
648 cs
->cst
->cs
.num_chunks
= 3;
652 cs
->cst
->flags
[0] = 0;
653 cs
->cst
->flags
[1] = RADEON_CS_RING_VCE
;
654 cs
->cst
->cs
.num_chunks
= 3;
660 cs
->cst
->flags
[0] = RADEON_CS_KEEP_TILING_FLAGS
;
661 cs
->cst
->flags
[1] = RADEON_CS_RING_GFX
;
662 cs
->cst
->cs
.num_chunks
= 3;
664 if (cs
->ws
->info
.r600_has_virtual_memory
) {
665 cs
->cst
->flags
[0] |= RADEON_CS_USE_VM
;
666 cs
->cst
->cs
.num_chunks
= 3;
668 if (flags
& PIPE_FLUSH_END_OF_FRAME
) {
669 cs
->cst
->flags
[0] |= RADEON_CS_END_OF_FRAME
;
670 cs
->cst
->cs
.num_chunks
= 3;
672 if (cs
->ring_type
== RING_COMPUTE
) {
673 cs
->cst
->flags
[1] = RADEON_CS_RING_COMPUTE
;
674 cs
->cst
->cs
.num_chunks
= 3;
679 if (util_queue_is_initialized(&cs
->ws
->cs_queue
)) {
680 util_queue_add_job(&cs
->ws
->cs_queue
, cs
, &cs
->flush_completed
,
681 radeon_drm_cs_emit_ioctl_oneshot
, NULL
);
682 if (!(flags
& PIPE_FLUSH_ASYNC
))
683 radeon_drm_cs_sync_flush(rcs
);
685 radeon_drm_cs_emit_ioctl_oneshot(cs
, 0);
688 radeon_cs_context_cleanup(cs
->cst
);
691 /* Prepare a new CS. */
692 cs
->base
.current
.buf
= cs
->csc
->buf
;
693 cs
->base
.current
.cdw
= 0;
694 cs
->base
.used_vram
= 0;
695 cs
->base
.used_gart
= 0;
697 if (cs
->ring_type
== RING_GFX
)
698 cs
->ws
->num_gfx_IBs
++;
699 else if (cs
->ring_type
== RING_DMA
)
700 cs
->ws
->num_sdma_IBs
++;
704 static void radeon_drm_cs_destroy(struct radeon_cmdbuf
*rcs
)
706 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
708 radeon_drm_cs_sync_flush(rcs
);
709 util_queue_fence_destroy(&cs
->flush_completed
);
710 radeon_cs_context_cleanup(&cs
->csc1
);
711 radeon_cs_context_cleanup(&cs
->csc2
);
712 p_atomic_dec(&cs
->ws
->num_cs
);
713 radeon_destroy_cs_context(&cs
->csc1
);
714 radeon_destroy_cs_context(&cs
->csc2
);
715 radeon_fence_reference(&cs
->next_fence
, NULL
);
719 static bool radeon_bo_is_referenced(struct radeon_cmdbuf
*rcs
,
720 struct pb_buffer
*_buf
,
721 enum radeon_bo_usage usage
)
723 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
724 struct radeon_bo
*bo
= (struct radeon_bo
*)_buf
;
727 if (!bo
->num_cs_references
)
730 index
= radeon_lookup_buffer(cs
->csc
, bo
);
735 index
= cs
->csc
->slab_buffers
[index
].u
.slab
.real_idx
;
737 if ((usage
& RADEON_USAGE_WRITE
) && cs
->csc
->relocs
[index
].write_domain
)
739 if ((usage
& RADEON_USAGE_READ
) && cs
->csc
->relocs
[index
].read_domains
)
747 static struct pipe_fence_handle
*
748 radeon_cs_create_fence(struct radeon_cmdbuf
*rcs
)
750 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
751 struct pb_buffer
*fence
;
753 /* Create a fence, which is a dummy BO. */
754 fence
= cs
->ws
->base
.buffer_create(&cs
->ws
->base
, 1, 1,
755 RADEON_DOMAIN_GTT
, RADEON_FLAG_NO_SUBALLOC
);
759 /* Add the fence as a dummy relocation. */
760 cs
->ws
->base
.cs_add_buffer(rcs
, fence
,
761 RADEON_USAGE_READWRITE
, RADEON_DOMAIN_GTT
,
763 return (struct pipe_fence_handle
*)fence
;
766 static bool radeon_fence_wait(struct radeon_winsys
*ws
,
767 struct pipe_fence_handle
*fence
,
770 return ws
->buffer_wait((struct pb_buffer
*)fence
, timeout
,
771 RADEON_USAGE_READWRITE
);
774 static void radeon_fence_reference(struct pipe_fence_handle
**dst
,
775 struct pipe_fence_handle
*src
)
777 pb_reference((struct pb_buffer
**)dst
, (struct pb_buffer
*)src
);
780 static struct pipe_fence_handle
*
781 radeon_drm_cs_get_next_fence(struct radeon_cmdbuf
*rcs
)
783 struct radeon_drm_cs
*cs
= radeon_drm_cs(rcs
);
784 struct pipe_fence_handle
*fence
= NULL
;
786 if (cs
->next_fence
) {
787 radeon_fence_reference(&fence
, cs
->next_fence
);
791 fence
= radeon_cs_create_fence(rcs
);
795 radeon_fence_reference(&cs
->next_fence
, fence
);
800 radeon_drm_cs_add_fence_dependency(struct radeon_cmdbuf
*cs
,
801 struct pipe_fence_handle
*fence
,
802 unsigned dependency_flags
)
804 /* TODO: Handle the following unlikely multi-threaded scenario:
806 * Thread 1 / Context 1 Thread 2 / Context 2
807 * -------------------- --------------------
808 * f = cs_get_next_fence()
809 * cs_add_fence_dependency(f)
813 * We currently assume that this does not happen because we don't support
814 * asynchronous flushes on Radeon.
818 void radeon_drm_cs_init_functions(struct radeon_drm_winsys
*ws
)
820 ws
->base
.ctx_create
= radeon_drm_ctx_create
;
821 ws
->base
.ctx_destroy
= radeon_drm_ctx_destroy
;
822 ws
->base
.cs_create
= radeon_drm_cs_create
;
823 ws
->base
.cs_destroy
= radeon_drm_cs_destroy
;
824 ws
->base
.cs_add_buffer
= radeon_drm_cs_add_buffer
;
825 ws
->base
.cs_lookup_buffer
= radeon_drm_cs_lookup_buffer
;
826 ws
->base
.cs_validate
= radeon_drm_cs_validate
;
827 ws
->base
.cs_check_space
= radeon_drm_cs_check_space
;
828 ws
->base
.cs_get_buffer_list
= radeon_drm_cs_get_buffer_list
;
829 ws
->base
.cs_flush
= radeon_drm_cs_flush
;
830 ws
->base
.cs_get_next_fence
= radeon_drm_cs_get_next_fence
;
831 ws
->base
.cs_is_buffer_referenced
= radeon_bo_is_referenced
;
832 ws
->base
.cs_sync_flush
= radeon_drm_cs_sync_flush
;
833 ws
->base
.cs_add_fence_dependency
= radeon_drm_cs_add_fence_dependency
;
834 ws
->base
.fence_wait
= radeon_fence_wait
;
835 ws
->base
.fence_reference
= radeon_fence_reference
;