projects / mesa.git / blob
? search:


c9502182a7ebee497e6da172eac6a6bc82b6eaa4
[mesa.git] / src / gallium / drivers / radeon / r600_pipe_common.c
1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include "r600_pipe_common.h"
25 #include "r600_cs.h"
26 #include "tgsi/tgsi_parse.h"
27 #include "util/list.h"
28 #include "util/u_draw_quad.h"
29 #include "util/u_memory.h"
30 #include "util/u_format_s3tc.h"
31 #include "util/u_upload_mgr.h"
32 #include "os/os_time.h"
33 #include "vl/vl_decoder.h"
34 #include "vl/vl_video_buffer.h"
35 #include "radeon/radeon_video.h"
36 #include "amd/common/ac_llvm_util.h"
37 #include "amd/common/sid.h"
38 #include <inttypes.h>
39 #include <sys/utsname.h>
40 #include <libsync.h>
41
42 #include <llvm-c/TargetMachine.h>
43
44
45 struct r600_multi_fence {
46 struct pipe_reference reference;
47 struct pipe_fence_handle *gfx;
48 struct pipe_fence_handle *sdma;
49
50 /* If the context wasn't flushed at fence creation, this is non-NULL. */
51 struct {
52 struct r600_common_context *ctx;
53 unsigned ib_index;
54 } gfx_unflushed;
55 };
56
57 /*
58 * shader binary helpers.
59 */
60 void si_radeon_shader_binary_init(struct ac_shader_binary *b)
61 {
62 memset(b, 0, sizeof(*b));
63 }
64
65 void si_radeon_shader_binary_clean(struct ac_shader_binary *b)
66 {
67 if (!b)
68 return;
69 FREE(b->code);
70 FREE(b->config);
71 FREE(b->rodata);
72 FREE(b->global_symbol_offsets);
73 FREE(b->relocs);
74 FREE(b->disasm_string);
75 FREE(b->llvm_ir_string);
76 }
77
78 /*
79 * pipe_context
80 */
81
82 /**
83 * Write an EOP event.
84 *
85 * \param event EVENT_TYPE_*
86 * \param event_flags Optional cache flush flags (TC)
87 * \param data_sel 1 = fence, 3 = timestamp
88 * \param buf Buffer
89 * \param va GPU address
90 * \param old_value Previous fence value (for a bug workaround)
91 * \param new_value Fence value to write for this event.
92 */
93 void si_gfx_write_event_eop(struct r600_common_context *ctx,
94 unsigned event, unsigned event_flags,
95 unsigned data_sel,
96 struct r600_resource *buf, uint64_t va,
97 uint32_t new_fence, unsigned query_type)
98 {
99 struct radeon_winsys_cs *cs = ctx->gfx.cs;
100 unsigned op = EVENT_TYPE(event) |
101 EVENT_INDEX(5) |
102 event_flags;
103 unsigned sel = EOP_DATA_SEL(data_sel);
104
105 /* Wait for write confirmation before writing data, but don't send
106 * an interrupt. */
107 if (data_sel != EOP_DATA_SEL_DISCARD)
108 sel |= EOP_INT_SEL(EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM);
109
110 if (ctx->chip_class >= GFX9) {
111 /* A ZPASS_DONE or PIXEL_STAT_DUMP_EVENT (of the DB occlusion
112 * counters) must immediately precede every timestamp event to
113 * prevent a GPU hang on GFX9.
114 *
115 * Occlusion queries don't need to do it here, because they
116 * always do ZPASS_DONE before the timestamp.
117 */
118 if (ctx->chip_class == GFX9 &&
119 query_type != PIPE_QUERY_OCCLUSION_COUNTER &&
120 query_type != PIPE_QUERY_OCCLUSION_PREDICATE &&
121 query_type != PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE) {
122 struct r600_resource *scratch = ctx->eop_bug_scratch;
123
124 assert(16 * ctx->screen->info.num_render_backends <=
125 scratch->b.b.width0);
126 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0));
127 radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1));
128 radeon_emit(cs, scratch->gpu_address);
129 radeon_emit(cs, scratch->gpu_address >> 32);
130
131 radeon_add_to_buffer_list(ctx, &ctx->gfx, scratch,
132 RADEON_USAGE_WRITE, RADEON_PRIO_QUERY);
133 }
134
135 radeon_emit(cs, PKT3(PKT3_RELEASE_MEM, 6, 0));
136 radeon_emit(cs, op);
137 radeon_emit(cs, sel);
138 radeon_emit(cs, va); /* address lo */
139 radeon_emit(cs, va >> 32); /* address hi */
140 radeon_emit(cs, new_fence); /* immediate data lo */
141 radeon_emit(cs, 0); /* immediate data hi */
142 radeon_emit(cs, 0); /* unused */
143 } else {
144 if (ctx->chip_class == CIK ||
145 ctx->chip_class == VI) {
146 struct r600_resource *scratch = ctx->eop_bug_scratch;
147 uint64_t va = scratch->gpu_address;
148
149 /* Two EOP events are required to make all engines go idle
150 * (and optional cache flushes executed) before the timestamp
151 * is written.
152 */
153 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, 0));
154 radeon_emit(cs, op);
155 radeon_emit(cs, va);
156 radeon_emit(cs, ((va >> 32) & 0xffff) | sel);
157 radeon_emit(cs, 0); /* immediate data */
158 radeon_emit(cs, 0); /* unused */
159
160 radeon_add_to_buffer_list(ctx, &ctx->gfx, scratch,
161 RADEON_USAGE_WRITE, RADEON_PRIO_QUERY);
162 }
163
164 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, 0));
165 radeon_emit(cs, op);
166 radeon_emit(cs, va);
167 radeon_emit(cs, ((va >> 32) & 0xffff) | sel);
168 radeon_emit(cs, new_fence); /* immediate data */
169 radeon_emit(cs, 0); /* unused */
170 }
171
172 if (buf) {
173 radeon_add_to_buffer_list(ctx, &ctx->gfx, buf, RADEON_USAGE_WRITE,
174 RADEON_PRIO_QUERY);
175 }
176 }
177
178 unsigned si_gfx_write_fence_dwords(struct r600_common_screen *screen)
179 {
180 unsigned dwords = 6;
181
182 if (screen->chip_class == CIK ||
183 screen->chip_class == VI)
184 dwords *= 2;
185
186 if (!screen->info.has_virtual_memory)
187 dwords += 2;
188
189 return dwords;
190 }
191
192 void si_gfx_wait_fence(struct r600_common_context *ctx,
193 uint64_t va, uint32_t ref, uint32_t mask)
194 {
195 struct radeon_winsys_cs *cs = ctx->gfx.cs;
196
197 radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0));
198 radeon_emit(cs, WAIT_REG_MEM_EQUAL | WAIT_REG_MEM_MEM_SPACE(1));
199 radeon_emit(cs, va);
200 radeon_emit(cs, va >> 32);
201 radeon_emit(cs, ref); /* reference value */
202 radeon_emit(cs, mask); /* mask */
203 radeon_emit(cs, 4); /* poll interval */
204 }
205
206 static void r600_dma_emit_wait_idle(struct r600_common_context *rctx)
207 {
208 struct radeon_winsys_cs *cs = rctx->dma.cs;
209
210 /* NOP waits for idle on Evergreen and later. */
211 if (rctx->chip_class >= CIK)
212 radeon_emit(cs, 0x00000000); /* NOP */
213 else
214 radeon_emit(cs, 0xf0000000); /* NOP */
215 }
216
217 void si_need_dma_space(struct r600_common_context *ctx, unsigned num_dw,
218 struct r600_resource *dst, struct r600_resource *src)
219 {
220 uint64_t vram = ctx->dma.cs->used_vram;
221 uint64_t gtt = ctx->dma.cs->used_gart;
222
223 if (dst) {
224 vram += dst->vram_usage;
225 gtt += dst->gart_usage;
226 }
227 if (src) {
228 vram += src->vram_usage;
229 gtt += src->gart_usage;
230 }
231
232 /* Flush the GFX IB if DMA depends on it. */
233 if (radeon_emitted(ctx->gfx.cs, ctx->initial_gfx_cs_size) &&
234 ((dst &&
235 ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, dst->buf,
236 RADEON_USAGE_READWRITE)) ||
237 (src &&
238 ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, src->buf,
239 RADEON_USAGE_WRITE))))
240 ctx->gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
241
242 /* Flush if there's not enough space, or if the memory usage per IB
243 * is too large.
244 *
245 * IBs using too little memory are limited by the IB submission overhead.
246 * IBs using too much memory are limited by the kernel/TTM overhead.
247 * Too long IBs create CPU-GPU pipeline bubbles and add latency.
248 *
249 * This heuristic makes sure that DMA requests are executed
250 * very soon after the call is made and lowers memory usage.
251 * It improves texture upload performance by keeping the DMA
252 * engine busy while uploads are being submitted.
253 */
254 num_dw++; /* for emit_wait_idle below */
255 if (!ctx->ws->cs_check_space(ctx->dma.cs, num_dw) ||
256 ctx->dma.cs->used_vram + ctx->dma.cs->used_gart > 64 * 1024 * 1024 ||
257 !radeon_cs_memory_below_limit(ctx->screen, ctx->dma.cs, vram, gtt)) {
258 ctx->dma.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
259 assert((num_dw + ctx->dma.cs->current.cdw) <= ctx->dma.cs->current.max_dw);
260 }
261
262 /* Wait for idle if either buffer has been used in the IB before to
263 * prevent read-after-write hazards.
264 */
265 if ((dst &&
266 ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, dst->buf,
267 RADEON_USAGE_READWRITE)) ||
268 (src &&
269 ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, src->buf,
270 RADEON_USAGE_WRITE)))
271 r600_dma_emit_wait_idle(ctx);
272
273 /* If GPUVM is not supported, the CS checker needs 2 entries
274 * in the buffer list per packet, which has to be done manually.
275 */
276 if (ctx->screen->info.has_virtual_memory) {
277 if (dst)
278 radeon_add_to_buffer_list(ctx, &ctx->dma, dst,
279 RADEON_USAGE_WRITE,
280 RADEON_PRIO_SDMA_BUFFER);
281 if (src)
282 radeon_add_to_buffer_list(ctx, &ctx->dma, src,
283 RADEON_USAGE_READ,
284 RADEON_PRIO_SDMA_BUFFER);
285 }
286
287 /* this function is called before all DMA calls, so increment this. */
288 ctx->num_dma_calls++;
289 }
290
291 static void r600_memory_barrier(struct pipe_context *ctx, unsigned flags)
292 {
293 }
294
295 void si_preflush_suspend_features(struct r600_common_context *ctx)
296 {
297 /* suspend queries */
298 if (!LIST_IS_EMPTY(&ctx->active_queries))
299 si_suspend_queries(ctx);
300 }
301
302 void si_postflush_resume_features(struct r600_common_context *ctx)
303 {
304 /* resume queries */
305 if (!LIST_IS_EMPTY(&ctx->active_queries))
306 si_resume_queries(ctx);
307 }
308
309 static void r600_add_fence_dependency(struct r600_common_context *rctx,
310 struct pipe_fence_handle *fence)
311 {
312 struct radeon_winsys *ws = rctx->ws;
313
314 if (rctx->dma.cs)
315 ws->cs_add_fence_dependency(rctx->dma.cs, fence);
316 ws->cs_add_fence_dependency(rctx->gfx.cs, fence);
317 }
318
319 static void r600_fence_server_sync(struct pipe_context *ctx,
320 struct pipe_fence_handle *fence)
321 {
322 struct r600_common_context *rctx = (struct r600_common_context *)ctx;
323 struct r600_multi_fence *rfence = (struct r600_multi_fence *)fence;
324
325 /* Only amdgpu needs to handle fence dependencies (for fence imports).
326 * radeon synchronizes all rings by default and will not implement
327 * fence imports.
328 */
329 if (rctx->screen->info.drm_major == 2)
330 return;
331
332 /* Only imported fences need to be handled by fence_server_sync,
333 * because the winsys handles synchronizations automatically for BOs
334 * within the process.
335 *
336 * Simply skip unflushed fences here, and the winsys will drop no-op
337 * dependencies (i.e. dependencies within the same ring).
338 */
339 if (rfence->gfx_unflushed.ctx)
340 return;
341
342 /* All unflushed commands will not start execution before
343 * this fence dependency is signalled.
344 *
345 * Should we flush the context to allow more GPU parallelism?
346 */
347 if (rfence->sdma)
348 r600_add_fence_dependency(rctx, rfence->sdma);
349 if (rfence->gfx)
350 r600_add_fence_dependency(rctx, rfence->gfx);
351 }
352
353 static void r600_create_fence_fd(struct pipe_context *ctx,
354 struct pipe_fence_handle **pfence, int fd)
355 {
356 struct r600_common_screen *rscreen = (struct r600_common_screen*)ctx->screen;
357 struct radeon_winsys *ws = rscreen->ws;
358 struct r600_multi_fence *rfence;
359
360 *pfence = NULL;
361
362 if (!rscreen->info.has_sync_file)
363 return;
364
365 rfence = CALLOC_STRUCT(r600_multi_fence);
366 if (!rfence)
367 return;
368
369 pipe_reference_init(&rfence->reference, 1);
370 rfence->gfx = ws->fence_import_sync_file(ws, fd);
371 if (!rfence->gfx) {
372 FREE(rfence);
373 return;
374 }
375
376 *pfence = (struct pipe_fence_handle*)rfence;
377 }
378
379 static int r600_fence_get_fd(struct pipe_screen *screen,
380 struct pipe_fence_handle *fence)
381 {
382 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
383 struct radeon_winsys *ws = rscreen->ws;
384 struct r600_multi_fence *rfence = (struct r600_multi_fence *)fence;
385 int gfx_fd = -1, sdma_fd = -1;
386
387 if (!rscreen->info.has_sync_file)
388 return -1;
389
390 /* Deferred fences aren't supported. */
391 assert(!rfence->gfx_unflushed.ctx);
392 if (rfence->gfx_unflushed.ctx)
393 return -1;
394
395 if (rfence->sdma) {
396 sdma_fd = ws->fence_export_sync_file(ws, rfence->sdma);
397 if (sdma_fd == -1)
398 return -1;
399 }
400 if (rfence->gfx) {
401 gfx_fd = ws->fence_export_sync_file(ws, rfence->gfx);
402 if (gfx_fd == -1) {
403 if (sdma_fd != -1)
404 close(sdma_fd);
405 return -1;
406 }
407 }
408
409 /* If we don't have FDs at this point, it means we don't have fences
410 * either. */
411 if (sdma_fd == -1)
412 return gfx_fd;
413 if (gfx_fd == -1)
414 return sdma_fd;
415
416 /* Get a fence that will be a combination of both fences. */
417 sync_accumulate("radeonsi", &gfx_fd, sdma_fd);
418 close(sdma_fd);
419 return gfx_fd;
420 }
421
422 static void r600_flush_from_st(struct pipe_context *ctx,
423 struct pipe_fence_handle **fence,
424 unsigned flags)
425 {
426 struct pipe_screen *screen = ctx->screen;
427 struct r600_common_context *rctx = (struct r600_common_context *)ctx;
428 struct radeon_winsys *ws = rctx->ws;
429 struct pipe_fence_handle *gfx_fence = NULL;
430 struct pipe_fence_handle *sdma_fence = NULL;
431 bool deferred_fence = false;
432 unsigned rflags = RADEON_FLUSH_ASYNC;
433
434 if (flags & PIPE_FLUSH_END_OF_FRAME)
435 rflags |= RADEON_FLUSH_END_OF_FRAME;
436
437 /* DMA IBs are preambles to gfx IBs, therefore must be flushed first. */
438 if (rctx->dma.cs)
439 rctx->dma.flush(rctx, rflags, fence ? &sdma_fence : NULL);
440
441 if (!radeon_emitted(rctx->gfx.cs, rctx->initial_gfx_cs_size)) {
442 if (fence)
443 ws->fence_reference(&gfx_fence, rctx->last_gfx_fence);
444 if (!(flags & PIPE_FLUSH_DEFERRED))
445 ws->cs_sync_flush(rctx->gfx.cs);
446 } else {
447 /* Instead of flushing, create a deferred fence. Constraints:
448 * - The state tracker must allow a deferred flush.
449 * - The state tracker must request a fence.
450 * - fence_get_fd is not allowed.
451 * Thread safety in fence_finish must be ensured by the state tracker.
452 */
453 if (flags & PIPE_FLUSH_DEFERRED &&
454 !(flags & PIPE_FLUSH_FENCE_FD) &&
455 fence) {
456 gfx_fence = rctx->ws->cs_get_next_fence(rctx->gfx.cs);
457 deferred_fence = true;
458 } else {
459 rctx->gfx.flush(rctx, rflags, fence ? &gfx_fence : NULL);
460 }
461 }
462
463 /* Both engines can signal out of order, so we need to keep both fences. */
464 if (fence) {
465 struct r600_multi_fence *multi_fence =
466 CALLOC_STRUCT(r600_multi_fence);
467 if (!multi_fence) {
468 ws->fence_reference(&sdma_fence, NULL);
469 ws->fence_reference(&gfx_fence, NULL);
470 goto finish;
471 }
472
473 multi_fence->reference.count = 1;
474 /* If both fences are NULL, fence_finish will always return true. */
475 multi_fence->gfx = gfx_fence;
476 multi_fence->sdma = sdma_fence;
477
478 if (deferred_fence) {
479 multi_fence->gfx_unflushed.ctx = rctx;
480 multi_fence->gfx_unflushed.ib_index = rctx->num_gfx_cs_flushes;
481 }
482
483 screen->fence_reference(screen, fence, NULL);
484 *fence = (struct pipe_fence_handle*)multi_fence;
485 }
486 finish:
487 if (!(flags & PIPE_FLUSH_DEFERRED)) {
488 if (rctx->dma.cs)
489 ws->cs_sync_flush(rctx->dma.cs);
490 ws->cs_sync_flush(rctx->gfx.cs);
491 }
492 }
493
494 static void r600_flush_dma_ring(void *ctx, unsigned flags,
495 struct pipe_fence_handle **fence)
496 {
497 struct r600_common_context *rctx = (struct r600_common_context *)ctx;
498 struct radeon_winsys_cs *cs = rctx->dma.cs;
499 struct radeon_saved_cs saved;
500 bool check_vm =
501 (rctx->screen->debug_flags & DBG(CHECK_VM)) &&
502 rctx->check_vm_faults;
503
504 if (!radeon_emitted(cs, 0)) {
505 if (fence)
506 rctx->ws->fence_reference(fence, rctx->last_sdma_fence);
507 return;
508 }
509
510 if (check_vm)
511 si_save_cs(rctx->ws, cs, &saved, true);
512
513 rctx->ws->cs_flush(cs, flags, &rctx->last_sdma_fence);
514 if (fence)
515 rctx->ws->fence_reference(fence, rctx->last_sdma_fence);
516
517 if (check_vm) {
518 /* Use conservative timeout 800ms, after which we won't wait any
519 * longer and assume the GPU is hung.
520 */
521 rctx->ws->fence_wait(rctx->ws, rctx->last_sdma_fence, 800*1000*1000);
522
523 rctx->check_vm_faults(rctx, &saved, RING_DMA);
524 si_clear_saved_cs(&saved);
525 }
526 }
527
528 /**
529 * Store a linearized copy of all chunks of \p cs together with the buffer
530 * list in \p saved.
531 */
532 void si_save_cs(struct radeon_winsys *ws, struct radeon_winsys_cs *cs,
533 struct radeon_saved_cs *saved, bool get_buffer_list)
534 {
535 uint32_t *buf;
536 unsigned i;
537
538 /* Save the IB chunks. */
539 saved->num_dw = cs->prev_dw + cs->current.cdw;
540 saved->ib = MALLOC(4 * saved->num_dw);
541 if (!saved->ib)
542 goto oom;
543
544 buf = saved->ib;
545 for (i = 0; i < cs->num_prev; ++i) {
546 memcpy(buf, cs->prev[i].buf, cs->prev[i].cdw * 4);
547 buf += cs->prev[i].cdw;
548 }
549 memcpy(buf, cs->current.buf, cs->current.cdw * 4);
550
551 if (!get_buffer_list)
552 return;
553
554 /* Save the buffer list. */
555 saved->bo_count = ws->cs_get_buffer_list(cs, NULL);
556 saved->bo_list = CALLOC(saved->bo_count,
557 sizeof(saved->bo_list[0]));
558 if (!saved->bo_list) {
559 FREE(saved->ib);
560 goto oom;
561 }
562 ws->cs_get_buffer_list(cs, saved->bo_list);
563
564 return;
565
566 oom:
567 fprintf(stderr, "%s: out of memory\n", __func__);
568 memset(saved, 0, sizeof(*saved));
569 }
570
571 void si_clear_saved_cs(struct radeon_saved_cs *saved)
572 {
573 FREE(saved->ib);
574 FREE(saved->bo_list);
575
576 memset(saved, 0, sizeof(*saved));
577 }
578
579 static enum pipe_reset_status r600_get_reset_status(struct pipe_context *ctx)
580 {
581 struct r600_common_context *rctx = (struct r600_common_context *)ctx;
582 unsigned latest = rctx->ws->query_value(rctx->ws,
583 RADEON_GPU_RESET_COUNTER);
584
585 if (rctx->gpu_reset_counter == latest)
586 return PIPE_NO_RESET;
587
588 rctx->gpu_reset_counter = latest;
589 return PIPE_UNKNOWN_CONTEXT_RESET;
590 }
591
592 static void r600_set_device_reset_callback(struct pipe_context *ctx,
593 const struct pipe_device_reset_callback *cb)
594 {
595 struct r600_common_context *rctx = (struct r600_common_context *)ctx;
596
597 if (cb)
598 rctx->device_reset_callback = *cb;
599 else
600 memset(&rctx->device_reset_callback, 0,
601 sizeof(rctx->device_reset_callback));
602 }
603
604 bool si_check_device_reset(struct r600_common_context *rctx)
605 {
606 enum pipe_reset_status status;
607
608 if (!rctx->device_reset_callback.reset)
609 return false;
610
611 if (!rctx->b.get_device_reset_status)
612 return false;
613
614 status = rctx->b.get_device_reset_status(&rctx->b);
615 if (status == PIPE_NO_RESET)
616 return false;
617
618 rctx->device_reset_callback.reset(rctx->device_reset_callback.data, status);
619 return true;
620 }
621
622 static void r600_dma_clear_buffer_fallback(struct pipe_context *ctx,
623 struct pipe_resource *dst,
624 uint64_t offset, uint64_t size,
625 unsigned value)
626 {
627 struct r600_common_context *rctx = (struct r600_common_context *)ctx;
628
629 rctx->clear_buffer(ctx, dst, offset, size, value, R600_COHERENCY_NONE);
630 }
631
632 static bool r600_resource_commit(struct pipe_context *pctx,
633 struct pipe_resource *resource,
634 unsigned level, struct pipe_box *box,
635 bool commit)
636 {
637 struct r600_common_context *ctx = (struct r600_common_context *)pctx;
638 struct r600_resource *res = r600_resource(resource);
639
640 /*
641 * Since buffer commitment changes cannot be pipelined, we need to
642 * (a) flush any pending commands that refer to the buffer we're about
643 * to change, and
644 * (b) wait for threaded submit to finish, including those that were
645 * triggered by some other, earlier operation.
646 */
647 if (radeon_emitted(ctx->gfx.cs, ctx->initial_gfx_cs_size) &&
648 ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs,
649 res->buf, RADEON_USAGE_READWRITE)) {
650 ctx->gfx.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
651 }
652 if (radeon_emitted(ctx->dma.cs, 0) &&
653 ctx->ws->cs_is_buffer_referenced(ctx->dma.cs,
654 res->buf, RADEON_USAGE_READWRITE)) {
655 ctx->dma.flush(ctx, RADEON_FLUSH_ASYNC, NULL);
656 }
657
658 ctx->ws->cs_sync_flush(ctx->dma.cs);
659 ctx->ws->cs_sync_flush(ctx->gfx.cs);
660
661 assert(resource->target == PIPE_BUFFER);
662
663 return ctx->ws->buffer_commit(res->buf, box->x, box->width, commit);
664 }
665
666 bool si_common_context_init(struct r600_common_context *rctx,
667 struct r600_common_screen *rscreen,
668 unsigned context_flags)
669 {
670 slab_create_child(&rctx->pool_transfers, &rscreen->pool_transfers);
671 slab_create_child(&rctx->pool_transfers_unsync, &rscreen->pool_transfers);
672
673 rctx->screen = rscreen;
674 rctx->ws = rscreen->ws;
675 rctx->family = rscreen->family;
676 rctx->chip_class = rscreen->chip_class;
677
678 rctx->b.invalidate_resource = si_invalidate_resource;
679 rctx->b.resource_commit = r600_resource_commit;
680 rctx->b.transfer_map = u_transfer_map_vtbl;
681 rctx->b.transfer_flush_region = u_transfer_flush_region_vtbl;
682 rctx->b.transfer_unmap = u_transfer_unmap_vtbl;
683 rctx->b.texture_subdata = u_default_texture_subdata;
684 rctx->b.memory_barrier = r600_memory_barrier;
685 rctx->b.flush = r600_flush_from_st;
686 rctx->b.create_fence_fd = r600_create_fence_fd;
687 rctx->b.fence_server_sync = r600_fence_server_sync;
688 rctx->dma_clear_buffer = r600_dma_clear_buffer_fallback;
689 rctx->b.buffer_subdata = si_buffer_subdata;
690
691 if (rscreen->info.drm_major == 2 && rscreen->info.drm_minor >= 43) {
692 rctx->b.get_device_reset_status = r600_get_reset_status;
693 rctx->gpu_reset_counter =
694 rctx->ws->query_value(rctx->ws,
695 RADEON_GPU_RESET_COUNTER);
696 }
697
698 rctx->b.set_device_reset_callback = r600_set_device_reset_callback;
699
700 si_init_context_texture_functions(rctx);
701 si_init_query_functions(rctx);
702
703 if (rctx->chip_class == CIK ||
704 rctx->chip_class == VI ||
705 rctx->chip_class == GFX9) {
706 rctx->eop_bug_scratch = (struct r600_resource*)
707 pipe_buffer_create(&rscreen->b, 0, PIPE_USAGE_DEFAULT,
708 16 * rscreen->info.num_render_backends);
709 if (!rctx->eop_bug_scratch)
710 return false;
711 }
712
713 rctx->allocator_zeroed_memory =
714 u_suballocator_create(&rctx->b, rscreen->info.gart_page_size,
715 0, PIPE_USAGE_DEFAULT, 0, true);
716 if (!rctx->allocator_zeroed_memory)
717 return false;
718
719 rctx->b.stream_uploader = u_upload_create(&rctx->b, 1024 * 1024,
720 0, PIPE_USAGE_STREAM);
721 if (!rctx->b.stream_uploader)
722 return false;
723
724 rctx->b.const_uploader = u_upload_create(&rctx->b, 128 * 1024,
725 0, PIPE_USAGE_DEFAULT);
726 if (!rctx->b.const_uploader)
727 return false;
728
729 rctx->ctx = rctx->ws->ctx_create(rctx->ws);
730 if (!rctx->ctx)
731 return false;
732
733 if (rscreen->info.num_sdma_rings && !(rscreen->debug_flags & DBG(NO_ASYNC_DMA))) {
734 rctx->dma.cs = rctx->ws->cs_create(rctx->ctx, RING_DMA,
735 r600_flush_dma_ring,
736 rctx);
737 rctx->dma.flush = r600_flush_dma_ring;
738 }
739
740 return true;
741 }
742
743 void si_common_context_cleanup(struct r600_common_context *rctx)
744 {
745 unsigned i,j;
746
747 /* Release DCC stats. */
748 for (i = 0; i < ARRAY_SIZE(rctx->dcc_stats); i++) {
749 assert(!rctx->dcc_stats[i].query_active);
750
751 for (j = 0; j < ARRAY_SIZE(rctx->dcc_stats[i].ps_stats); j++)
752 if (rctx->dcc_stats[i].ps_stats[j])
753 rctx->b.destroy_query(&rctx->b,
754 rctx->dcc_stats[i].ps_stats[j]);
755
756 r600_texture_reference(&rctx->dcc_stats[i].tex, NULL);
757 }
758
759 if (rctx->query_result_shader)
760 rctx->b.delete_compute_state(&rctx->b, rctx->query_result_shader);
761
762 if (rctx->gfx.cs)
763 rctx->ws->cs_destroy(rctx->gfx.cs);
764 if (rctx->dma.cs)
765 rctx->ws->cs_destroy(rctx->dma.cs);
766 if (rctx->ctx)
767 rctx->ws->ctx_destroy(rctx->ctx);
768
769 if (rctx->b.stream_uploader)
770 u_upload_destroy(rctx->b.stream_uploader);
771 if (rctx->b.const_uploader)
772 u_upload_destroy(rctx->b.const_uploader);
773
774 slab_destroy_child(&rctx->pool_transfers);
775 slab_destroy_child(&rctx->pool_transfers_unsync);
776
777 if (rctx->allocator_zeroed_memory) {
778 u_suballocator_destroy(rctx->allocator_zeroed_memory);
779 }
780 rctx->ws->fence_reference(&rctx->last_gfx_fence, NULL);
781 rctx->ws->fence_reference(&rctx->last_sdma_fence, NULL);
782 r600_resource_reference(&rctx->eop_bug_scratch, NULL);
783 }
784
785 /*
786 * pipe_screen
787 */
788
789 static const struct debug_named_value common_debug_options[] = {
790 /* logging */
791 { "tex", DBG(TEX), "Print texture info" },
792 { "nir", DBG(NIR), "Enable experimental NIR shaders" },
793 { "compute", DBG(COMPUTE), "Print compute info" },
794 { "vm", DBG(VM), "Print virtual addresses when creating resources" },
795 { "info", DBG(INFO), "Print driver information" },
796
797 /* shaders */
798 { "vs", DBG(VS), "Print vertex shaders" },
799 { "gs", DBG(GS), "Print geometry shaders" },
800 { "ps", DBG(PS), "Print pixel shaders" },
801 { "cs", DBG(CS), "Print compute shaders" },
802 { "tcs", DBG(TCS), "Print tessellation control shaders" },
803 { "tes", DBG(TES), "Print tessellation evaluation shaders" },
804 { "noir", DBG(NO_IR), "Don't print the LLVM IR"},
805 { "notgsi", DBG(NO_TGSI), "Don't print the TGSI"},
806 { "noasm", DBG(NO_ASM), "Don't print disassembled shaders"},
807 { "preoptir", DBG(PREOPT_IR), "Print the LLVM IR before initial optimizations" },
808 { "checkir", DBG(CHECK_IR), "Enable additional sanity checks on shader IR" },
809 { "nooptvariant", DBG(NO_OPT_VARIANT), "Disable compiling optimized shader variants." },
810
811 { "testdma", DBG(TEST_DMA), "Invoke SDMA tests and exit." },
812 { "testvmfaultcp", DBG(TEST_VMFAULT_CP), "Invoke a CP VM fault test and exit." },
813 { "testvmfaultsdma", DBG(TEST_VMFAULT_SDMA), "Invoke a SDMA VM fault test and exit." },
814 { "testvmfaultshader", DBG(TEST_VMFAULT_SHADER), "Invoke a shader VM fault test and exit." },
815
816 /* features */
817 { "nodma", DBG(NO_ASYNC_DMA), "Disable asynchronous DMA" },
818 { "nohyperz", DBG(NO_HYPERZ), "Disable Hyper-Z" },
819 /* GL uses the word INVALIDATE, gallium uses the word DISCARD */
820 { "noinvalrange", DBG(NO_DISCARD_RANGE), "Disable handling of INVALIDATE_RANGE map flags" },
821 { "no2d", DBG(NO_2D_TILING), "Disable 2D tiling" },
822 { "notiling", DBG(NO_TILING), "Disable tiling" },
823 { "switch_on_eop", DBG(SWITCH_ON_EOP), "Program WD/IA to switch on end-of-packet." },
824 { "forcedma", DBG(FORCE_DMA), "Use asynchronous DMA for all operations when possible." },
825 { "precompile", DBG(PRECOMPILE), "Compile one shader variant at shader creation." },
826 { "nowc", DBG(NO_WC), "Disable GTT write combining" },
827 { "check_vm", DBG(CHECK_VM), "Check VM faults and dump debug info." },
828 { "nodcc", DBG(NO_DCC), "Disable DCC." },
829 { "nodccclear", DBG(NO_DCC_CLEAR), "Disable DCC fast clear." },
830 { "norbplus", DBG(NO_RB_PLUS), "Disable RB+." },
831 { "sisched", DBG(SI_SCHED), "Enable LLVM SI Machine Instruction Scheduler." },
832 { "mono", DBG(MONOLITHIC_SHADERS), "Use old-style monolithic shaders compiled on demand" },
833 { "unsafemath", DBG(UNSAFE_MATH), "Enable unsafe math shader optimizations" },
834 { "nodccfb", DBG(NO_DCC_FB), "Disable separate DCC on the main framebuffer" },
835 { "nodpbb", DBG(NO_DPBB), "Disable DPBB." },
836 { "nodfsm", DBG(NO_DFSM), "Disable DFSM." },
837 { "dpbb", DBG(DPBB), "Enable DPBB." },
838 { "dfsm", DBG(DFSM), "Enable DFSM." },
839 { "nooutoforder", DBG(NO_OUT_OF_ORDER), "Disable out-of-order rasterization" },
840 { "reserve_vmid", DBG(RESERVE_VMID), "Force VMID reservation per context." },
841
842 DEBUG_NAMED_VALUE_END /* must be last */
843 };
844
845 static const char* r600_get_vendor(struct pipe_screen* pscreen)
846 {
847 return "X.Org";
848 }
849
850 static const char* r600_get_device_vendor(struct pipe_screen* pscreen)
851 {
852 return "AMD";
853 }
854
855 static const char *r600_get_marketing_name(struct radeon_winsys *ws)
856 {
857 if (!ws->get_chip_name)
858 return NULL;
859 return ws->get_chip_name(ws);
860 }
861
862 static const char *r600_get_family_name(const struct r600_common_screen *rscreen)
863 {
864 switch (rscreen->info.family) {
865 case CHIP_TAHITI: return "AMD TAHITI";
866 case CHIP_PITCAIRN: return "AMD PITCAIRN";
867 case CHIP_VERDE: return "AMD CAPE VERDE";
868 case CHIP_OLAND: return "AMD OLAND";
869 case CHIP_HAINAN: return "AMD HAINAN";
870 case CHIP_BONAIRE: return "AMD BONAIRE";
871 case CHIP_KAVERI: return "AMD KAVERI";
872 case CHIP_KABINI: return "AMD KABINI";
873 case CHIP_HAWAII: return "AMD HAWAII";
874 case CHIP_MULLINS: return "AMD MULLINS";
875 case CHIP_TONGA: return "AMD TONGA";
876 case CHIP_ICELAND: return "AMD ICELAND";
877 case CHIP_CARRIZO: return "AMD CARRIZO";
878 case CHIP_FIJI: return "AMD FIJI";
879 case CHIP_POLARIS10: return "AMD POLARIS10";
880 case CHIP_POLARIS11: return "AMD POLARIS11";
881 case CHIP_POLARIS12: return "AMD POLARIS12";
882 case CHIP_STONEY: return "AMD STONEY";
883 case CHIP_VEGA10: return "AMD VEGA10";
884 case CHIP_RAVEN: return "AMD RAVEN";
885 default: return "AMD unknown";
886 }
887 }
888
889 static void r600_disk_cache_create(struct r600_common_screen *rscreen)
890 {
891 /* Don't use the cache if shader dumping is enabled. */
892 if (rscreen->debug_flags & DBG_ALL_SHADERS)
893 return;
894
895 uint32_t mesa_timestamp;
896 if (disk_cache_get_function_timestamp(r600_disk_cache_create,
897 &mesa_timestamp)) {
898 char *timestamp_str;
899 int res = -1;
900 uint32_t llvm_timestamp;
901
902 if (disk_cache_get_function_timestamp(LLVMInitializeAMDGPUTargetInfo,
903 &llvm_timestamp)) {
904 res = asprintf(&timestamp_str, "%u_%u",
905 mesa_timestamp, llvm_timestamp);
906 }
907
908 if (res != -1) {
909 /* These flags affect shader compilation. */
910 uint64_t shader_debug_flags =
911 rscreen->debug_flags &
912 (DBG(FS_CORRECT_DERIVS_AFTER_KILL) |
913 DBG(SI_SCHED) |
914 DBG(UNSAFE_MATH));
915
916 rscreen->disk_shader_cache =
917 disk_cache_create(r600_get_family_name(rscreen),
918 timestamp_str,
919 shader_debug_flags);
920 free(timestamp_str);
921 }
922 }
923 }
924
925 static struct disk_cache *r600_get_disk_shader_cache(struct pipe_screen *pscreen)
926 {
927 struct r600_common_screen *rscreen = (struct r600_common_screen*)pscreen;
928 return rscreen->disk_shader_cache;
929 }
930
931 static const char* r600_get_name(struct pipe_screen* pscreen)
932 {
933 struct r600_common_screen *rscreen = (struct r600_common_screen*)pscreen;
934
935 return rscreen->renderer_string;
936 }
937
938 static float r600_get_paramf(struct pipe_screen* pscreen,
939 enum pipe_capf param)
940 {
941 switch (param) {
942 case PIPE_CAPF_MAX_LINE_WIDTH:
943 case PIPE_CAPF_MAX_LINE_WIDTH_AA:
944 case PIPE_CAPF_MAX_POINT_WIDTH:
945 case PIPE_CAPF_MAX_POINT_WIDTH_AA:
946 return 8192.0f;
947 case PIPE_CAPF_MAX_TEXTURE_ANISOTROPY:
948 return 16.0f;
949 case PIPE_CAPF_MAX_TEXTURE_LOD_BIAS:
950 return 16.0f;
951 case PIPE_CAPF_GUARD_BAND_LEFT:
952 case PIPE_CAPF_GUARD_BAND_TOP:
953 case PIPE_CAPF_GUARD_BAND_RIGHT:
954 case PIPE_CAPF_GUARD_BAND_BOTTOM:
955 return 0.0f;
956 }
957 return 0.0f;
958 }
959
960 static int r600_get_video_param(struct pipe_screen *screen,
961 enum pipe_video_profile profile,
962 enum pipe_video_entrypoint entrypoint,
963 enum pipe_video_cap param)
964 {
965 switch (param) {
966 case PIPE_VIDEO_CAP_SUPPORTED:
967 return vl_profile_supported(screen, profile, entrypoint);
968 case PIPE_VIDEO_CAP_NPOT_TEXTURES:
969 return 1;
970 case PIPE_VIDEO_CAP_MAX_WIDTH:
971 case PIPE_VIDEO_CAP_MAX_HEIGHT:
972 return vl_video_buffer_max_size(screen);
973 case PIPE_VIDEO_CAP_PREFERED_FORMAT:
974 return PIPE_FORMAT_NV12;
975 case PIPE_VIDEO_CAP_PREFERS_INTERLACED:
976 return false;
977 case PIPE_VIDEO_CAP_SUPPORTS_INTERLACED:
978 return false;
979 case PIPE_VIDEO_CAP_SUPPORTS_PROGRESSIVE:
980 return true;
981 case PIPE_VIDEO_CAP_MAX_LEVEL:
982 return vl_level_supported(screen, profile);
983 default:
984 return 0;
985 }
986 }
987
988 static unsigned get_max_threads_per_block(struct r600_common_screen *screen,
989 enum pipe_shader_ir ir_type)
990 {
991 if (ir_type != PIPE_SHADER_IR_TGSI)
992 return 256;
993
994 /* Only 16 waves per thread-group on gfx9. */
995 if (screen->chip_class >= GFX9)
996 return 1024;
997
998 /* Up to 40 waves per thread-group on GCN < gfx9. Expose a nice
999 * round number.
1000 */
1001 return 2048;
1002 }
1003
1004 static int r600_get_compute_param(struct pipe_screen *screen,
1005 enum pipe_shader_ir ir_type,
1006 enum pipe_compute_cap param,
1007 void *ret)
1008 {
1009 struct r600_common_screen *rscreen = (struct r600_common_screen *)screen;
1010
1011 //TODO: select these params by asic
1012 switch (param) {
1013 case PIPE_COMPUTE_CAP_IR_TARGET: {
1014 const char *gpu;
1015 const char *triple;
1016
1017 if (HAVE_LLVM < 0x0400)
1018 triple = "amdgcn--";
1019 else
1020 triple = "amdgcn-mesa-mesa3d";
1021
1022 gpu = ac_get_llvm_processor_name(rscreen->family);
1023 if (ret) {
1024 sprintf(ret, "%s-%s", gpu, triple);
1025 }
1026 /* +2 for dash and terminating NIL byte */
1027 return (strlen(triple) + strlen(gpu) + 2) * sizeof(char);
1028 }
1029 case PIPE_COMPUTE_CAP_GRID_DIMENSION:
1030 if (ret) {
1031 uint64_t *grid_dimension = ret;
1032 grid_dimension[0] = 3;
1033 }
1034 return 1 * sizeof(uint64_t);
1035
1036 case PIPE_COMPUTE_CAP_MAX_GRID_SIZE:
1037 if (ret) {
1038 uint64_t *grid_size = ret;
1039 grid_size[0] = 65535;
1040 grid_size[1] = 65535;
1041 grid_size[2] = 65535;
1042 }
1043 return 3 * sizeof(uint64_t) ;
1044
1045 case PIPE_COMPUTE_CAP_MAX_BLOCK_SIZE:
1046 if (ret) {
1047 uint64_t *block_size = ret;
1048 unsigned threads_per_block = get_max_threads_per_block(rscreen, ir_type);
1049 block_size[0] = threads_per_block;
1050 block_size[1] = threads_per_block;
1051 block_size[2] = threads_per_block;
1052 }
1053 return 3 * sizeof(uint64_t);
1054
1055 case PIPE_COMPUTE_CAP_MAX_THREADS_PER_BLOCK:
1056 if (ret) {
1057 uint64_t *max_threads_per_block = ret;
1058 *max_threads_per_block = get_max_threads_per_block(rscreen, ir_type);
1059 }
1060 return sizeof(uint64_t);
1061 case PIPE_COMPUTE_CAP_ADDRESS_BITS:
1062 if (ret) {
1063 uint32_t *address_bits = ret;
1064 address_bits[0] = 64;
1065 }
1066 return 1 * sizeof(uint32_t);
1067
1068 case PIPE_COMPUTE_CAP_MAX_GLOBAL_SIZE:
1069 if (ret) {
1070 uint64_t *max_global_size = ret;
1071 uint64_t max_mem_alloc_size;
1072
1073 r600_get_compute_param(screen, ir_type,
1074 PIPE_COMPUTE_CAP_MAX_MEM_ALLOC_SIZE,
1075 &max_mem_alloc_size);
1076
1077 /* In OpenCL, the MAX_MEM_ALLOC_SIZE must be at least
1078 * 1/4 of the MAX_GLOBAL_SIZE. Since the
1079 * MAX_MEM_ALLOC_SIZE is fixed for older kernels,
1080 * make sure we never report more than
1081 * 4 * MAX_MEM_ALLOC_SIZE.
1082 */
1083 *max_global_size = MIN2(4 * max_mem_alloc_size,
1084 MAX2(rscreen->info.gart_size,
1085 rscreen->info.vram_size));
1086 }
1087 return sizeof(uint64_t);
1088
1089 case PIPE_COMPUTE_CAP_MAX_LOCAL_SIZE:
1090 if (ret) {
1091 uint64_t *max_local_size = ret;
1092 /* Value reported by the closed source driver. */
1093 *max_local_size = 32768;
1094 }
1095 return sizeof(uint64_t);
1096
1097 case PIPE_COMPUTE_CAP_MAX_INPUT_SIZE:
1098 if (ret) {
1099 uint64_t *max_input_size = ret;
1100 /* Value reported by the closed source driver. */
1101 *max_input_size = 1024;
1102 }
1103 return sizeof(uint64_t);
1104
1105 case PIPE_COMPUTE_CAP_MAX_MEM_ALLOC_SIZE:
1106 if (ret) {
1107 uint64_t *max_mem_alloc_size = ret;
1108
1109 *max_mem_alloc_size = rscreen->info.max_alloc_size;
1110 }
1111 return sizeof(uint64_t);
1112
1113 case PIPE_COMPUTE_CAP_MAX_CLOCK_FREQUENCY:
1114 if (ret) {
1115 uint32_t *max_clock_frequency = ret;
1116 *max_clock_frequency = rscreen->info.max_shader_clock;
1117 }
1118 return sizeof(uint32_t);
1119
1120 case PIPE_COMPUTE_CAP_MAX_COMPUTE_UNITS:
1121 if (ret) {
1122 uint32_t *max_compute_units = ret;
1123 *max_compute_units = rscreen->info.num_good_compute_units;
1124 }
1125 return sizeof(uint32_t);
1126
1127 case PIPE_COMPUTE_CAP_IMAGES_SUPPORTED:
1128 if (ret) {
1129 uint32_t *images_supported = ret;
1130 *images_supported = 0;
1131 }
1132 return sizeof(uint32_t);
1133 case PIPE_COMPUTE_CAP_MAX_PRIVATE_SIZE:
1134 break; /* unused */
1135 case PIPE_COMPUTE_CAP_SUBGROUP_SIZE:
1136 if (ret) {
1137 uint32_t *subgroup_size = ret;
1138 *subgroup_size = 64;
1139 }
1140 return sizeof(uint32_t);
1141 case PIPE_COMPUTE_CAP_MAX_VARIABLE_THREADS_PER_BLOCK:
1142 if (ret) {
1143 uint64_t *max_variable_threads_per_block = ret;
1144 if (ir_type == PIPE_SHADER_IR_TGSI)
1145 *max_variable_threads_per_block = SI_MAX_VARIABLE_THREADS_PER_BLOCK;
1146 else
1147 *max_variable_threads_per_block = 0;
1148 }
1149 return sizeof(uint64_t);
1150 }
1151
1152 fprintf(stderr, "unknown PIPE_COMPUTE_CAP %d\n", param);
1153 return 0;
1154 }
1155
1156 static uint64_t r600_get_timestamp(struct pipe_screen *screen)
1157 {
1158 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
1159
1160 return 1000000 * rscreen->ws->query_value(rscreen->ws, RADEON_TIMESTAMP) /
1161 rscreen->info.clock_crystal_freq;
1162 }
1163
1164 static void r600_fence_reference(struct pipe_screen *screen,
1165 struct pipe_fence_handle **dst,
1166 struct pipe_fence_handle *src)
1167 {
1168 struct radeon_winsys *ws = ((struct r600_common_screen*)screen)->ws;
1169 struct r600_multi_fence **rdst = (struct r600_multi_fence **)dst;
1170 struct r600_multi_fence *rsrc = (struct r600_multi_fence *)src;
1171
1172 if (pipe_reference(&(*rdst)->reference, &rsrc->reference)) {
1173 ws->fence_reference(&(*rdst)->gfx, NULL);
1174 ws->fence_reference(&(*rdst)->sdma, NULL);
1175 FREE(*rdst);
1176 }
1177 *rdst = rsrc;
1178 }
1179
1180 static boolean r600_fence_finish(struct pipe_screen *screen,
1181 struct pipe_context *ctx,
1182 struct pipe_fence_handle *fence,
1183 uint64_t timeout)
1184 {
1185 struct radeon_winsys *rws = ((struct r600_common_screen*)screen)->ws;
1186 struct r600_multi_fence *rfence = (struct r600_multi_fence *)fence;
1187 struct r600_common_context *rctx;
1188 int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
1189
1190 ctx = threaded_context_unwrap_sync(ctx);
1191 rctx = ctx ? (struct r600_common_context*)ctx : NULL;
1192
1193 if (rfence->sdma) {
1194 if (!rws->fence_wait(rws, rfence->sdma, timeout))
1195 return false;
1196
1197 /* Recompute the timeout after waiting. */
1198 if (timeout && timeout != PIPE_TIMEOUT_INFINITE) {
1199 int64_t time = os_time_get_nano();
1200 timeout = abs_timeout > time ? abs_timeout - time : 0;
1201 }
1202 }
1203
1204 if (!rfence->gfx)
1205 return true;
1206
1207 /* Flush the gfx IB if it hasn't been flushed yet. */
1208 if (rctx &&
1209 rfence->gfx_unflushed.ctx == rctx &&
1210 rfence->gfx_unflushed.ib_index == rctx->num_gfx_cs_flushes) {
1211 rctx->gfx.flush(rctx, timeout ? 0 : RADEON_FLUSH_ASYNC, NULL);
1212 rfence->gfx_unflushed.ctx = NULL;
1213
1214 if (!timeout)
1215 return false;
1216
1217 /* Recompute the timeout after all that. */
1218 if (timeout && timeout != PIPE_TIMEOUT_INFINITE) {
1219 int64_t time = os_time_get_nano();
1220 timeout = abs_timeout > time ? abs_timeout - time : 0;
1221 }
1222 }
1223
1224 return rws->fence_wait(rws, rfence->gfx, timeout);
1225 }
1226
1227 static void r600_query_memory_info(struct pipe_screen *screen,
1228 struct pipe_memory_info *info)
1229 {
1230 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen;
1231 struct radeon_winsys *ws = rscreen->ws;
1232 unsigned vram_usage, gtt_usage;
1233
1234 info->total_device_memory = rscreen->info.vram_size / 1024;
1235 info->total_staging_memory = rscreen->info.gart_size / 1024;
1236
1237 /* The real TTM memory usage is somewhat random, because:
1238 *
1239 * 1) TTM delays freeing memory, because it can only free it after
1240 * fences expire.
1241 *
1242 * 2) The memory usage can be really low if big VRAM evictions are
1243 * taking place, but the real usage is well above the size of VRAM.
1244 *
1245 * Instead, return statistics of this process.
1246 */
1247 vram_usage = ws->query_value(ws, RADEON_REQUESTED_VRAM_MEMORY) / 1024;
1248 gtt_usage = ws->query_value(ws, RADEON_REQUESTED_GTT_MEMORY) / 1024;
1249
1250 info->avail_device_memory =
1251 vram_usage <= info->total_device_memory ?
1252 info->total_device_memory - vram_usage : 0;
1253 info->avail_staging_memory =
1254 gtt_usage <= info->total_staging_memory ?
1255 info->total_staging_memory - gtt_usage : 0;
1256
1257 info->device_memory_evicted =
1258 ws->query_value(ws, RADEON_NUM_BYTES_MOVED) / 1024;
1259
1260 if (rscreen->info.drm_major == 3 && rscreen->info.drm_minor >= 4)
1261 info->nr_device_memory_evictions =
1262 ws->query_value(ws, RADEON_NUM_EVICTIONS);
1263 else
1264 /* Just return the number of evicted 64KB pages. */
1265 info->nr_device_memory_evictions = info->device_memory_evicted / 64;
1266 }
1267
1268 struct pipe_resource *si_resource_create_common(struct pipe_screen *screen,
1269 const struct pipe_resource *templ)
1270 {
1271 if (templ->target == PIPE_BUFFER) {
1272 return si_buffer_create(screen, templ, 256);
1273 } else {
1274 return si_texture_create(screen, templ);
1275 }
1276 }
1277
1278 bool si_common_screen_init(struct r600_common_screen *rscreen,
1279 struct radeon_winsys *ws)
1280 {
1281 char family_name[32] = {}, llvm_string[32] = {}, kernel_version[128] = {};
1282 struct utsname uname_data;
1283 const char *chip_name;
1284
1285 ws->query_info(ws, &rscreen->info);
1286 rscreen->ws = ws;
1287
1288 if ((chip_name = r600_get_marketing_name(ws)))
1289 snprintf(family_name, sizeof(family_name), "%s / ",
1290 r600_get_family_name(rscreen) + 4);
1291 else
1292 chip_name = r600_get_family_name(rscreen);
1293
1294 if (uname(&uname_data) == 0)
1295 snprintf(kernel_version, sizeof(kernel_version),
1296 " / %s", uname_data.release);
1297
1298 if (HAVE_LLVM > 0) {
1299 snprintf(llvm_string, sizeof(llvm_string),
1300 ", LLVM %i.%i.%i", (HAVE_LLVM >> 8) & 0xff,
1301 HAVE_LLVM & 0xff, MESA_LLVM_VERSION_PATCH);
1302 }
1303
1304 snprintf(rscreen->renderer_string, sizeof(rscreen->renderer_string),
1305 "%s (%sDRM %i.%i.%i%s%s)",
1306 chip_name, family_name, rscreen->info.drm_major,
1307 rscreen->info.drm_minor, rscreen->info.drm_patchlevel,
1308 kernel_version, llvm_string);
1309
1310 rscreen->b.get_name = r600_get_name;
1311 rscreen->b.get_vendor = r600_get_vendor;
1312 rscreen->b.get_device_vendor = r600_get_device_vendor;
1313 rscreen->b.get_disk_shader_cache = r600_get_disk_shader_cache;
1314 rscreen->b.get_compute_param = r600_get_compute_param;
1315 rscreen->b.get_paramf = r600_get_paramf;
1316 rscreen->b.get_timestamp = r600_get_timestamp;
1317 rscreen->b.fence_finish = r600_fence_finish;
1318 rscreen->b.fence_reference = r600_fence_reference;
1319 rscreen->b.resource_destroy = u_resource_destroy_vtbl;
1320 rscreen->b.resource_from_user_memory = si_buffer_from_user_memory;
1321 rscreen->b.query_memory_info = r600_query_memory_info;
1322 rscreen->b.fence_get_fd = r600_fence_get_fd;
1323
1324 if (rscreen->info.has_hw_decode) {
1325 rscreen->b.get_video_param = si_vid_get_video_param;
1326 rscreen->b.is_video_format_supported = si_vid_is_format_supported;
1327 } else {
1328 rscreen->b.get_video_param = r600_get_video_param;
1329 rscreen->b.is_video_format_supported = vl_video_buffer_is_format_supported;
1330 }
1331
1332 si_init_screen_texture_functions(rscreen);
1333 si_init_screen_query_functions(rscreen);
1334
1335 rscreen->family = rscreen->info.family;
1336 rscreen->chip_class = rscreen->info.chip_class;
1337 rscreen->debug_flags |= debug_get_flags_option("R600_DEBUG", common_debug_options, 0);
1338 rscreen->has_rbplus = false;
1339 rscreen->rbplus_allowed = false;
1340
1341 r600_disk_cache_create(rscreen);
1342
1343 slab_create_parent(&rscreen->pool_transfers, sizeof(struct r600_transfer), 64);
1344
1345 rscreen->force_aniso = MIN2(16, debug_get_num_option("R600_TEX_ANISO", -1));
1346 if (rscreen->force_aniso >= 0) {
1347 printf("radeon: Forcing anisotropy filter to %ix\n",
1348 /* round down to a power of two */
1349 1 << util_logbase2(rscreen->force_aniso));
1350 }
1351
1352 (void) mtx_init(&rscreen->aux_context_lock, mtx_plain);
1353 (void) mtx_init(&rscreen->gpu_load_mutex, mtx_plain);
1354
1355 if (rscreen->debug_flags & DBG(INFO)) {
1356 printf("pci (domain:bus:dev.func): %04x:%02x:%02x.%x\n",
1357 rscreen->info.pci_domain, rscreen->info.pci_bus,
1358 rscreen->info.pci_dev, rscreen->info.pci_func);
1359 printf("pci_id = 0x%x\n", rscreen->info.pci_id);
1360 printf("family = %i (%s)\n", rscreen->info.family,
1361 r600_get_family_name(rscreen));
1362 printf("chip_class = %i\n", rscreen->info.chip_class);
1363 printf("pte_fragment_size = %u\n", rscreen->info.pte_fragment_size);
1364 printf("gart_page_size = %u\n", rscreen->info.gart_page_size);
1365 printf("gart_size = %i MB\n", (int)DIV_ROUND_UP(rscreen->info.gart_size, 1024*1024));
1366 printf("vram_size = %i MB\n", (int)DIV_ROUND_UP(rscreen->info.vram_size, 1024*1024));
1367 printf("vram_vis_size = %i MB\n", (int)DIV_ROUND_UP(rscreen->info.vram_vis_size, 1024*1024));
1368 printf("max_alloc_size = %i MB\n",
1369 (int)DIV_ROUND_UP(rscreen->info.max_alloc_size, 1024*1024));
1370 printf("min_alloc_size = %u\n", rscreen->info.min_alloc_size);
1371 printf("has_dedicated_vram = %u\n", rscreen->info.has_dedicated_vram);
1372 printf("has_virtual_memory = %i\n", rscreen->info.has_virtual_memory);
1373 printf("gfx_ib_pad_with_type2 = %i\n", rscreen->info.gfx_ib_pad_with_type2);
1374 printf("has_hw_decode = %u\n", rscreen->info.has_hw_decode);
1375 printf("num_sdma_rings = %i\n", rscreen->info.num_sdma_rings);
1376 printf("num_compute_rings = %u\n", rscreen->info.num_compute_rings);
1377 printf("uvd_fw_version = %u\n", rscreen->info.uvd_fw_version);
1378 printf("vce_fw_version = %u\n", rscreen->info.vce_fw_version);
1379 printf("me_fw_version = %i\n", rscreen->info.me_fw_version);
1380 printf("me_fw_feature = %i\n", rscreen->info.me_fw_feature);
1381 printf("pfp_fw_version = %i\n", rscreen->info.pfp_fw_version);
1382 printf("pfp_fw_feature = %i\n", rscreen->info.pfp_fw_feature);
1383 printf("ce_fw_version = %i\n", rscreen->info.ce_fw_version);
1384 printf("ce_fw_feature = %i\n", rscreen->info.ce_fw_feature);
1385 printf("vce_harvest_config = %i\n", rscreen->info.vce_harvest_config);
1386 printf("clock_crystal_freq = %i\n", rscreen->info.clock_crystal_freq);
1387 printf("tcc_cache_line_size = %u\n", rscreen->info.tcc_cache_line_size);
1388 printf("drm = %i.%i.%i\n", rscreen->info.drm_major,
1389 rscreen->info.drm_minor, rscreen->info.drm_patchlevel);
1390 printf("has_userptr = %i\n", rscreen->info.has_userptr);
1391 printf("has_syncobj = %u\n", rscreen->info.has_syncobj);
1392 printf("has_sync_file = %u\n", rscreen->info.has_sync_file);
1393
1394 printf("r600_max_quad_pipes = %i\n", rscreen->info.r600_max_quad_pipes);
1395 printf("max_shader_clock = %i\n", rscreen->info.max_shader_clock);
1396 printf("num_good_compute_units = %i\n", rscreen->info.num_good_compute_units);
1397 printf("max_se = %i\n", rscreen->info.max_se);
1398 printf("max_sh_per_se = %i\n", rscreen->info.max_sh_per_se);
1399
1400 printf("r600_gb_backend_map = %i\n", rscreen->info.r600_gb_backend_map);
1401 printf("r600_gb_backend_map_valid = %i\n", rscreen->info.r600_gb_backend_map_valid);
1402 printf("r600_num_banks = %i\n", rscreen->info.r600_num_banks);
1403 printf("num_render_backends = %i\n", rscreen->info.num_render_backends);
1404 printf("num_tile_pipes = %i\n", rscreen->info.num_tile_pipes);
1405 printf("pipe_interleave_bytes = %i\n", rscreen->info.pipe_interleave_bytes);
1406 printf("enabled_rb_mask = 0x%x\n", rscreen->info.enabled_rb_mask);
1407 printf("max_alignment = %u\n", (unsigned)rscreen->info.max_alignment);
1408 }
1409 return true;
1410 }
1411
1412 void si_destroy_common_screen(struct r600_common_screen *rscreen)
1413 {
1414 si_perfcounters_destroy(rscreen);
1415 si_gpu_load_kill_thread(rscreen);
1416
1417 mtx_destroy(&rscreen->gpu_load_mutex);
1418 mtx_destroy(&rscreen->aux_context_lock);
1419 rscreen->aux_context->destroy(rscreen->aux_context);
1420
1421 slab_destroy_parent(&rscreen->pool_transfers);
1422
1423 disk_cache_destroy(rscreen->disk_shader_cache);
1424 rscreen->ws->destroy(rscreen->ws);
1425 FREE(rscreen);
1426 }
1427
1428 bool si_can_dump_shader(struct r600_common_screen *rscreen,
1429 unsigned processor)
1430 {
1431 return rscreen->debug_flags & (1 << processor);
1432 }
1433
1434 bool si_extra_shader_checks(struct r600_common_screen *rscreen, unsigned processor)
1435 {
1436 return (rscreen->debug_flags & DBG(CHECK_IR)) ||
1437 si_can_dump_shader(rscreen, processor);
1438 }
1439
1440 void si_screen_clear_buffer(struct r600_common_screen *rscreen, struct pipe_resource *dst,
1441 uint64_t offset, uint64_t size, unsigned value)
1442 {
1443 struct r600_common_context *rctx = (struct r600_common_context*)rscreen->aux_context;
1444
1445 mtx_lock(&rscreen->aux_context_lock);
1446 rctx->dma_clear_buffer(&rctx->b, dst, offset, size, value);
1447 rscreen->aux_context->flush(rscreen->aux_context, NULL, 0);
1448 mtx_unlock(&rscreen->aux_context_lock);
1449 }