r600g: fix tiling with cayman and virtual memory
[mesa.git] / src / gallium / winsys / radeon / drm / radeon_drm_bo.c
1 /*
2 * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
14 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
15 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
16 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
17 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * The above copyright notice and this permission notice (including the
23 * next paragraph) shall be included in all copies or substantial portions
24 * of the Software.
25 */
26
27 #define _FILE_OFFSET_BITS 64
28 #include "radeon_drm_cs.h"
29
30 #include "util/u_hash_table.h"
31 #include "util/u_memory.h"
32 #include "util/u_simple_list.h"
33 #include "util/u_double_list.h"
34 #include "os/os_thread.h"
35 #include "os/os_mman.h"
36
37 #include "state_tracker/drm_driver.h"
38
39 #include <sys/ioctl.h>
40 #include <xf86drm.h>
41 #include <errno.h>
42
43 /*
44 * this are copy from radeon_drm, once an updated libdrm is released
45 * we should bump configure.ac requirement for it and remove the following
46 * field
47 */
48 #define RADEON_BO_FLAGS_MACRO_TILE 1
49 #define RADEON_BO_FLAGS_MICRO_TILE 2
50 #define RADEON_BO_FLAGS_MICRO_TILE_SQUARE 0x20
51
52 #ifndef DRM_RADEON_GEM_WAIT
53 #define DRM_RADEON_GEM_WAIT 0x2b
54
55 #define RADEON_GEM_NO_WAIT 0x1
56 #define RADEON_GEM_USAGE_READ 0x2
57 #define RADEON_GEM_USAGE_WRITE 0x4
58
59 struct drm_radeon_gem_wait {
60 uint32_t handle;
61 uint32_t flags; /* one of RADEON_GEM_* */
62 };
63
64 #endif
65
66 #ifndef RADEON_VA_MAP
67
68 #define RADEON_VA_MAP 1
69 #define RADEON_VA_UNMAP 2
70
71 #define RADEON_VA_RESULT_OK 0
72 #define RADEON_VA_RESULT_ERROR 1
73 #define RADEON_VA_RESULT_VA_EXIST 2
74
75 #define RADEON_VM_PAGE_VALID (1 << 0)
76 #define RADEON_VM_PAGE_READABLE (1 << 1)
77 #define RADEON_VM_PAGE_WRITEABLE (1 << 2)
78 #define RADEON_VM_PAGE_SYSTEM (1 << 3)
79 #define RADEON_VM_PAGE_SNOOPED (1 << 4)
80
81 struct drm_radeon_gem_va {
82 uint32_t handle;
83 uint32_t operation;
84 uint32_t vm_id;
85 uint32_t flags;
86 uint64_t offset;
87 };
88
89 #define DRM_RADEON_GEM_VA 0x2b
90 #endif
91
92
93
94 extern const struct pb_vtbl radeon_bo_vtbl;
95
96
97 static INLINE struct radeon_bo *radeon_bo(struct pb_buffer *bo)
98 {
99 assert(bo->vtbl == &radeon_bo_vtbl);
100 return (struct radeon_bo *)bo;
101 }
102
103 struct radeon_bo_va_hole {
104 struct list_head list;
105 uint64_t offset;
106 uint64_t size;
107 };
108
109 struct radeon_bomgr {
110 /* Base class. */
111 struct pb_manager base;
112
113 /* Winsys. */
114 struct radeon_drm_winsys *rws;
115
116 /* List of buffer handles and its mutex. */
117 struct util_hash_table *bo_handles;
118 pipe_mutex bo_handles_mutex;
119 pipe_mutex bo_va_mutex;
120
121 /* is virtual address supported */
122 bool va;
123 unsigned va_offset;
124 struct list_head va_holes;
125 };
126
127 static INLINE struct radeon_bomgr *radeon_bomgr(struct pb_manager *mgr)
128 {
129 return (struct radeon_bomgr *)mgr;
130 }
131
132 static struct radeon_bo *get_radeon_bo(struct pb_buffer *_buf)
133 {
134 struct radeon_bo *bo = NULL;
135
136 if (_buf->vtbl == &radeon_bo_vtbl) {
137 bo = radeon_bo(_buf);
138 } else {
139 struct pb_buffer *base_buf;
140 pb_size offset;
141 pb_get_base_buffer(_buf, &base_buf, &offset);
142
143 if (base_buf->vtbl == &radeon_bo_vtbl)
144 bo = radeon_bo(base_buf);
145 }
146
147 return bo;
148 }
149
150 static void radeon_bo_wait(struct pb_buffer *_buf, enum radeon_bo_usage usage)
151 {
152 struct radeon_bo *bo = get_radeon_bo(_buf);
153
154 while (p_atomic_read(&bo->num_active_ioctls)) {
155 sched_yield();
156 }
157
158 /* XXX use this when it's ready */
159 /*if (bo->rws->info.drm_minor >= 12) {
160 struct drm_radeon_gem_wait args = {};
161 args.handle = bo->handle;
162 args.flags = usage;
163 while (drmCommandWriteRead(bo->rws->fd, DRM_RADEON_GEM_WAIT,
164 &args, sizeof(args)) == -EBUSY);
165 } else*/ {
166 struct drm_radeon_gem_wait_idle args;
167 memset(&args, 0, sizeof(args));
168 args.handle = bo->handle;
169 while (drmCommandWriteRead(bo->rws->fd, DRM_RADEON_GEM_WAIT_IDLE,
170 &args, sizeof(args)) == -EBUSY);
171 }
172 }
173
174 static boolean radeon_bo_is_busy(struct pb_buffer *_buf,
175 enum radeon_bo_usage usage)
176 {
177 struct radeon_bo *bo = get_radeon_bo(_buf);
178
179 if (p_atomic_read(&bo->num_active_ioctls)) {
180 return TRUE;
181 }
182
183 /* XXX use this when it's ready */
184 /*if (bo->rws->info.drm_minor >= 12) {
185 struct drm_radeon_gem_wait args = {};
186 args.handle = bo->handle;
187 args.flags = usage | RADEON_GEM_NO_WAIT;
188 return drmCommandWriteRead(bo->rws->fd, DRM_RADEON_GEM_WAIT,
189 &args, sizeof(args)) != 0;
190 } else*/ {
191 struct drm_radeon_gem_busy args;
192 memset(&args, 0, sizeof(args));
193 args.handle = bo->handle;
194 return drmCommandWriteRead(bo->rws->fd, DRM_RADEON_GEM_BUSY,
195 &args, sizeof(args)) != 0;
196 }
197 }
198
199 static uint64_t radeon_bomgr_find_va(struct radeon_bomgr *mgr, uint64_t size, uint64_t alignment)
200 {
201 struct radeon_bo_va_hole *hole, *n;
202 uint64_t offset = 0, waste = 0;
203
204 pipe_mutex_lock(mgr->bo_va_mutex);
205 /* first look for a hole */
206 LIST_FOR_EACH_ENTRY_SAFE(hole, n, &mgr->va_holes, list) {
207 offset = hole->offset;
208 waste = 0;
209 if (alignment) {
210 waste = offset % alignment;
211 waste = waste ? alignment - waste : 0;
212 }
213 offset += waste;
214 if (!waste && hole->size == size) {
215 offset = hole->offset;
216 list_del(&hole->list);
217 FREE(hole);
218 pipe_mutex_unlock(mgr->bo_va_mutex);
219 return offset;
220 }
221 if ((hole->size - waste) >= size) {
222 if (waste) {
223 n = CALLOC_STRUCT(radeon_bo_va_hole);
224 n->size = waste;
225 n->offset = hole->offset;
226 list_add(&n->list, &mgr->va_holes);
227 }
228 hole->size -= (size + waste);
229 hole->offset += size + waste;
230 pipe_mutex_unlock(mgr->bo_va_mutex);
231 return offset;
232 }
233 }
234
235 offset = mgr->va_offset;
236 waste = 0;
237 if (alignment) {
238 waste = offset % alignment;
239 waste = waste ? alignment - waste : 0;
240 }
241 offset += waste;
242 mgr->va_offset += size + waste;
243 pipe_mutex_unlock(mgr->bo_va_mutex);
244 return offset;
245 }
246
247 static void radeon_bomgr_force_va(struct radeon_bomgr *mgr, uint64_t va, uint64_t size)
248 {
249 pipe_mutex_lock(mgr->bo_va_mutex);
250 if (va >= mgr->va_offset) {
251 if (va > mgr->va_offset) {
252 struct radeon_bo_va_hole *hole;
253 hole = CALLOC_STRUCT(radeon_bo_va_hole);
254 if (hole) {
255 hole->size = va - mgr->va_offset;
256 hole->offset = mgr->va_offset;
257 list_add(&hole->list, &mgr->va_holes);
258 }
259 }
260 mgr->va_offset = va + size;
261 } else {
262 struct radeon_bo_va_hole *hole, *n;
263 uint64_t stmp, etmp;
264
265 /* free all holes that fall into the range
266 * NOTE that we might lose virtual address space
267 */
268 LIST_FOR_EACH_ENTRY_SAFE(hole, n, &mgr->va_holes, list) {
269 stmp = hole->offset;
270 etmp = stmp + hole->size;
271 if (va >= stmp && va < etmp) {
272 list_del(&hole->list);
273 FREE(hole);
274 }
275 }
276 }
277 pipe_mutex_unlock(mgr->bo_va_mutex);
278 }
279
280 static void radeon_bomgr_free_va(struct radeon_bomgr *mgr, uint64_t va, uint64_t size)
281 {
282 pipe_mutex_lock(mgr->bo_va_mutex);
283 if ((va + size) == mgr->va_offset) {
284 mgr->va_offset = va;
285 } else {
286 struct radeon_bo_va_hole *hole;
287
288 /* FIXME on allocation failure we just lose virtual address space
289 * maybe print a warning
290 */
291 hole = CALLOC_STRUCT(radeon_bo_va_hole);
292 if (hole) {
293 hole->size = size;
294 hole->offset = va;
295 list_add(&hole->list, &mgr->va_holes);
296 }
297 }
298 pipe_mutex_unlock(mgr->bo_va_mutex);
299 }
300
301 static void radeon_bo_destroy(struct pb_buffer *_buf)
302 {
303 struct radeon_bo *bo = radeon_bo(_buf);
304 struct radeon_bomgr *mgr = bo->mgr;
305 struct drm_gem_close args;
306
307 memset(&args, 0, sizeof(args));
308
309 if (bo->name) {
310 pipe_mutex_lock(bo->mgr->bo_handles_mutex);
311 util_hash_table_remove(bo->mgr->bo_handles,
312 (void*)(uintptr_t)bo->name);
313 pipe_mutex_unlock(bo->mgr->bo_handles_mutex);
314 }
315
316 if (bo->ptr)
317 os_munmap(bo->ptr, bo->base.size);
318
319 if (mgr->va) {
320 radeon_bomgr_free_va(mgr, bo->va, bo->va_size);
321 }
322
323 /* Close object. */
324 args.handle = bo->handle;
325 drmIoctl(bo->rws->fd, DRM_IOCTL_GEM_CLOSE, &args);
326 pipe_mutex_destroy(bo->map_mutex);
327 FREE(bo);
328 }
329
330 static unsigned get_pb_usage_from_transfer_flags(enum pipe_transfer_usage usage)
331 {
332 unsigned res = 0;
333
334 if (usage & PIPE_TRANSFER_WRITE)
335 res |= PB_USAGE_CPU_WRITE;
336
337 if (usage & PIPE_TRANSFER_DONTBLOCK)
338 res |= PB_USAGE_DONTBLOCK;
339
340 if (usage & PIPE_TRANSFER_UNSYNCHRONIZED)
341 res |= PB_USAGE_UNSYNCHRONIZED;
342
343 return res;
344 }
345
346 static void *radeon_bo_map_internal(struct pb_buffer *_buf,
347 unsigned flags, void *flush_ctx)
348 {
349 struct radeon_bo *bo = radeon_bo(_buf);
350 struct radeon_drm_cs *cs = flush_ctx;
351 struct drm_radeon_gem_mmap args;
352 void *ptr;
353
354 memset(&args, 0, sizeof(args));
355
356 /* If it's not unsynchronized bo_map, flush CS if needed and then wait. */
357 if (!(flags & PB_USAGE_UNSYNCHRONIZED)) {
358 /* DONTBLOCK doesn't make sense with UNSYNCHRONIZED. */
359 if (flags & PB_USAGE_DONTBLOCK) {
360 if (!(flags & PB_USAGE_CPU_WRITE)) {
361 /* Mapping for read.
362 *
363 * Since we are mapping for read, we don't need to wait
364 * if the GPU is using the buffer for read too
365 * (neither one is changing it).
366 *
367 * Only check whether the buffer is being used for write. */
368 if (radeon_bo_is_referenced_by_cs_for_write(cs, bo)) {
369 cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC);
370 return NULL;
371 }
372
373 if (radeon_bo_is_busy((struct pb_buffer*)bo,
374 RADEON_USAGE_WRITE)) {
375 return NULL;
376 }
377 } else {
378 if (radeon_bo_is_referenced_by_cs(cs, bo)) {
379 cs->flush_cs(cs->flush_data, RADEON_FLUSH_ASYNC);
380 return NULL;
381 }
382
383 if (radeon_bo_is_busy((struct pb_buffer*)bo,
384 RADEON_USAGE_READWRITE)) {
385 return NULL;
386 }
387 }
388 } else {
389 if (!(flags & PB_USAGE_CPU_WRITE)) {
390 /* Mapping for read.
391 *
392 * Since we are mapping for read, we don't need to wait
393 * if the GPU is using the buffer for read too
394 * (neither one is changing it).
395 *
396 * Only check whether the buffer is being used for write. */
397 if (radeon_bo_is_referenced_by_cs_for_write(cs, bo)) {
398 cs->flush_cs(cs->flush_data, 0);
399 }
400 radeon_bo_wait((struct pb_buffer*)bo,
401 RADEON_USAGE_WRITE);
402 } else {
403 /* Mapping for write. */
404 if (radeon_bo_is_referenced_by_cs(cs, bo)) {
405 cs->flush_cs(cs->flush_data, 0);
406 } else {
407 /* Try to avoid busy-waiting in radeon_bo_wait. */
408 if (p_atomic_read(&bo->num_active_ioctls))
409 radeon_drm_cs_sync_flush(cs);
410 }
411
412 radeon_bo_wait((struct pb_buffer*)bo, RADEON_USAGE_READWRITE);
413 }
414 }
415 }
416
417 /* Return the pointer if it's already mapped. */
418 if (bo->ptr)
419 return bo->ptr;
420
421 /* Map the buffer. */
422 pipe_mutex_lock(bo->map_mutex);
423 /* Return the pointer if it's already mapped (in case of a race). */
424 if (bo->ptr) {
425 pipe_mutex_unlock(bo->map_mutex);
426 return bo->ptr;
427 }
428 args.handle = bo->handle;
429 args.offset = 0;
430 args.size = (uint64_t)bo->base.size;
431 if (drmCommandWriteRead(bo->rws->fd,
432 DRM_RADEON_GEM_MMAP,
433 &args,
434 sizeof(args))) {
435 pipe_mutex_unlock(bo->map_mutex);
436 fprintf(stderr, "radeon: gem_mmap failed: %p 0x%08X\n",
437 bo, bo->handle);
438 return NULL;
439 }
440
441 ptr = os_mmap(0, args.size, PROT_READ|PROT_WRITE, MAP_SHARED,
442 bo->rws->fd, args.addr_ptr);
443 if (ptr == MAP_FAILED) {
444 pipe_mutex_unlock(bo->map_mutex);
445 fprintf(stderr, "radeon: mmap failed, errno: %i\n", errno);
446 return NULL;
447 }
448 bo->ptr = ptr;
449 pipe_mutex_unlock(bo->map_mutex);
450
451 return bo->ptr;
452 }
453
454 static void radeon_bo_unmap_internal(struct pb_buffer *_buf)
455 {
456 /* NOP */
457 }
458
459 static void radeon_bo_get_base_buffer(struct pb_buffer *buf,
460 struct pb_buffer **base_buf,
461 unsigned *offset)
462 {
463 *base_buf = buf;
464 *offset = 0;
465 }
466
467 static enum pipe_error radeon_bo_validate(struct pb_buffer *_buf,
468 struct pb_validate *vl,
469 unsigned flags)
470 {
471 /* Always pinned */
472 return PIPE_OK;
473 }
474
475 static void radeon_bo_fence(struct pb_buffer *buf,
476 struct pipe_fence_handle *fence)
477 {
478 }
479
480 const struct pb_vtbl radeon_bo_vtbl = {
481 radeon_bo_destroy,
482 radeon_bo_map_internal,
483 radeon_bo_unmap_internal,
484 radeon_bo_validate,
485 radeon_bo_fence,
486 radeon_bo_get_base_buffer,
487 };
488
489 static struct pb_buffer *radeon_bomgr_create_bo(struct pb_manager *_mgr,
490 pb_size size,
491 const struct pb_desc *desc)
492 {
493 struct radeon_bomgr *mgr = radeon_bomgr(_mgr);
494 struct radeon_drm_winsys *rws = mgr->rws;
495 struct radeon_bo *bo;
496 struct drm_radeon_gem_create args;
497 struct radeon_bo_desc *rdesc = (struct radeon_bo_desc*)desc;
498 int r;
499
500 memset(&args, 0, sizeof(args));
501
502 assert(rdesc->initial_domains);
503 assert((rdesc->initial_domains &
504 ~(RADEON_GEM_DOMAIN_GTT | RADEON_GEM_DOMAIN_VRAM)) == 0);
505
506 args.size = size;
507 args.alignment = desc->alignment;
508 args.initial_domain = rdesc->initial_domains;
509
510 if (drmCommandWriteRead(rws->fd, DRM_RADEON_GEM_CREATE,
511 &args, sizeof(args))) {
512 fprintf(stderr, "radeon: Failed to allocate a buffer:\n");
513 fprintf(stderr, "radeon: size : %d bytes\n", size);
514 fprintf(stderr, "radeon: alignment : %d bytes\n", desc->alignment);
515 fprintf(stderr, "radeon: domains : %d\n", args.initial_domain);
516 return NULL;
517 }
518
519 bo = CALLOC_STRUCT(radeon_bo);
520 if (!bo)
521 return NULL;
522
523 pipe_reference_init(&bo->base.reference, 1);
524 bo->base.alignment = desc->alignment;
525 bo->base.usage = desc->usage;
526 bo->base.size = size;
527 bo->base.vtbl = &radeon_bo_vtbl;
528 bo->mgr = mgr;
529 bo->rws = mgr->rws;
530 bo->handle = args.handle;
531 bo->va = 0;
532 pipe_mutex_init(bo->map_mutex);
533
534 if (mgr->va) {
535 struct drm_radeon_gem_va va;
536
537 bo->va_size = align(size, 4096);
538 bo->va = radeon_bomgr_find_va(mgr, bo->va_size, desc->alignment);
539
540 va.handle = bo->handle;
541 va.vm_id = 0;
542 va.operation = RADEON_VA_MAP;
543 va.flags = RADEON_VM_PAGE_READABLE |
544 RADEON_VM_PAGE_WRITEABLE |
545 RADEON_VM_PAGE_SNOOPED;
546 va.offset = bo->va;
547 r = drmCommandWriteRead(rws->fd, DRM_RADEON_GEM_VA, &va, sizeof(va));
548 if (r && va.operation == RADEON_VA_RESULT_ERROR) {
549 fprintf(stderr, "radeon: Failed to allocate a buffer:\n");
550 fprintf(stderr, "radeon: size : %d bytes\n", size);
551 fprintf(stderr, "radeon: alignment : %d bytes\n", desc->alignment);
552 fprintf(stderr, "radeon: domains : %d\n", args.initial_domain);
553 radeon_bo_destroy(&bo->base);
554 return NULL;
555 }
556 if (va.operation == RADEON_VA_RESULT_VA_EXIST) {
557 radeon_bomgr_free_va(mgr, bo->va, bo->va_size);
558 bo->va = va.offset;
559 radeon_bomgr_force_va(mgr, bo->va, bo->va_size);
560 }
561 }
562
563 return &bo->base;
564 }
565
566 static void radeon_bomgr_flush(struct pb_manager *mgr)
567 {
568 /* NOP */
569 }
570
571 /* This is for the cache bufmgr. */
572 static boolean radeon_bomgr_is_buffer_busy(struct pb_manager *_mgr,
573 struct pb_buffer *_buf)
574 {
575 struct radeon_bo *bo = radeon_bo(_buf);
576
577 if (radeon_bo_is_referenced_by_any_cs(bo)) {
578 return TRUE;
579 }
580
581 if (radeon_bo_is_busy((struct pb_buffer*)bo, RADEON_USAGE_READWRITE)) {
582 return TRUE;
583 }
584
585 return FALSE;
586 }
587
588 static void radeon_bomgr_destroy(struct pb_manager *_mgr)
589 {
590 struct radeon_bomgr *mgr = radeon_bomgr(_mgr);
591 util_hash_table_destroy(mgr->bo_handles);
592 pipe_mutex_destroy(mgr->bo_handles_mutex);
593 pipe_mutex_destroy(mgr->bo_va_mutex);
594 FREE(mgr);
595 }
596
597 #define PTR_TO_UINT(x) ((unsigned)((intptr_t)(x)))
598
599 static unsigned handle_hash(void *key)
600 {
601 return PTR_TO_UINT(key);
602 }
603
604 static int handle_compare(void *key1, void *key2)
605 {
606 return PTR_TO_UINT(key1) != PTR_TO_UINT(key2);
607 }
608
609 struct pb_manager *radeon_bomgr_create(struct radeon_drm_winsys *rws)
610 {
611 struct radeon_bomgr *mgr;
612
613 mgr = CALLOC_STRUCT(radeon_bomgr);
614 if (!mgr)
615 return NULL;
616
617 mgr->base.destroy = radeon_bomgr_destroy;
618 mgr->base.create_buffer = radeon_bomgr_create_bo;
619 mgr->base.flush = radeon_bomgr_flush;
620 mgr->base.is_buffer_busy = radeon_bomgr_is_buffer_busy;
621
622 mgr->rws = rws;
623 mgr->bo_handles = util_hash_table_create(handle_hash, handle_compare);
624 pipe_mutex_init(mgr->bo_handles_mutex);
625 pipe_mutex_init(mgr->bo_va_mutex);
626
627 mgr->va = rws->info.r600_virtual_address;
628 mgr->va_offset = rws->info.r600_va_start;
629 list_inithead(&mgr->va_holes);
630
631 return &mgr->base;
632 }
633
634 static void *radeon_bo_map(struct pb_buffer *buf,
635 struct radeon_winsys_cs *cs,
636 enum pipe_transfer_usage usage)
637 {
638 return pb_map(buf, get_pb_usage_from_transfer_flags(usage), cs);
639 }
640
641 static unsigned eg_tile_split(unsigned tile_split)
642 {
643 switch (tile_split) {
644 case 0: tile_split = 64; break;
645 case 1: tile_split = 128; break;
646 case 2: tile_split = 256; break;
647 case 3: tile_split = 512; break;
648 default:
649 case 4: tile_split = 1024; break;
650 case 5: tile_split = 2048; break;
651 case 6: tile_split = 4096; break;
652 }
653 return tile_split;
654 }
655
656 static void radeon_bo_get_tiling(struct pb_buffer *_buf,
657 enum radeon_bo_layout *microtiled,
658 enum radeon_bo_layout *macrotiled,
659 unsigned *bankw, unsigned *bankh,
660 unsigned *tile_split,
661 unsigned *stencil_tile_split,
662 unsigned *mtilea)
663 {
664 struct radeon_bo *bo = get_radeon_bo(_buf);
665 struct drm_radeon_gem_set_tiling args;
666
667 memset(&args, 0, sizeof(args));
668
669 args.handle = bo->handle;
670
671 drmCommandWriteRead(bo->rws->fd,
672 DRM_RADEON_GEM_GET_TILING,
673 &args,
674 sizeof(args));
675
676 *microtiled = RADEON_LAYOUT_LINEAR;
677 *macrotiled = RADEON_LAYOUT_LINEAR;
678 if (args.tiling_flags & RADEON_BO_FLAGS_MICRO_TILE)
679 *microtiled = RADEON_LAYOUT_TILED;
680
681 if (args.tiling_flags & RADEON_BO_FLAGS_MACRO_TILE)
682 *macrotiled = RADEON_LAYOUT_TILED;
683 if (bankw && tile_split && stencil_tile_split && mtilea && tile_split) {
684 *bankw = (args.tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
685 *bankh = (args.tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
686 *tile_split = (args.tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
687 *stencil_tile_split = (args.tiling_flags >> RADEON_TILING_EG_STENCIL_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_STENCIL_TILE_SPLIT_MASK;
688 *mtilea = (args.tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
689 *tile_split = eg_tile_split(*tile_split);
690 }
691 }
692
693 static void radeon_bo_set_tiling(struct pb_buffer *_buf,
694 struct radeon_winsys_cs *rcs,
695 enum radeon_bo_layout microtiled,
696 enum radeon_bo_layout macrotiled,
697 uint32_t pitch)
698 {
699 struct radeon_bo *bo = get_radeon_bo(_buf);
700 struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
701 struct drm_radeon_gem_set_tiling args;
702
703 memset(&args, 0, sizeof(args));
704
705 /* Tiling determines how DRM treats the buffer data.
706 * We must flush CS when changing it if the buffer is referenced. */
707 if (cs && radeon_bo_is_referenced_by_cs(cs, bo)) {
708 cs->flush_cs(cs->flush_data, 0);
709 }
710
711 while (p_atomic_read(&bo->num_active_ioctls)) {
712 sched_yield();
713 }
714
715 if (microtiled == RADEON_LAYOUT_TILED)
716 args.tiling_flags |= RADEON_BO_FLAGS_MICRO_TILE;
717 else if (microtiled == RADEON_LAYOUT_SQUARETILED)
718 args.tiling_flags |= RADEON_BO_FLAGS_MICRO_TILE_SQUARE;
719
720 if (macrotiled == RADEON_LAYOUT_TILED)
721 args.tiling_flags |= RADEON_BO_FLAGS_MACRO_TILE;
722
723 args.handle = bo->handle;
724 args.pitch = pitch;
725
726 drmCommandWriteRead(bo->rws->fd,
727 DRM_RADEON_GEM_SET_TILING,
728 &args,
729 sizeof(args));
730 }
731
732 static struct radeon_winsys_cs_handle *radeon_drm_get_cs_handle(
733 struct pb_buffer *_buf)
734 {
735 /* return radeon_bo. */
736 return (struct radeon_winsys_cs_handle*)get_radeon_bo(_buf);
737 }
738
739 static struct pb_buffer *
740 radeon_winsys_bo_create(struct radeon_winsys *rws,
741 unsigned size,
742 unsigned alignment,
743 unsigned bind,
744 enum radeon_bo_domain domain)
745 {
746 struct radeon_drm_winsys *ws = radeon_drm_winsys(rws);
747 struct radeon_bo_desc desc;
748 struct pb_manager *provider;
749 struct pb_buffer *buffer;
750
751 memset(&desc, 0, sizeof(desc));
752 desc.base.alignment = alignment;
753
754 /* Additional criteria for the cache manager. */
755 desc.base.usage = domain;
756 desc.initial_domains = domain;
757
758 /* Assign a buffer manager. */
759 if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER |
760 PIPE_BIND_CONSTANT_BUFFER | PIPE_BIND_CUSTOM))
761 provider = ws->cman;
762 else
763 provider = ws->kman;
764
765 buffer = provider->create_buffer(provider, size, &desc.base);
766 if (!buffer)
767 return NULL;
768
769 return (struct pb_buffer*)buffer;
770 }
771
772 static struct pb_buffer *radeon_winsys_bo_from_handle(struct radeon_winsys *rws,
773 struct winsys_handle *whandle,
774 unsigned *stride)
775 {
776 struct radeon_drm_winsys *ws = radeon_drm_winsys(rws);
777 struct radeon_bo *bo;
778 struct radeon_bomgr *mgr = radeon_bomgr(ws->kman);
779 struct drm_gem_open open_arg = {};
780 int r;
781
782 memset(&open_arg, 0, sizeof(open_arg));
783
784 /* We must maintain a list of pairs <handle, bo>, so that we always return
785 * the same BO for one particular handle. If we didn't do that and created
786 * more than one BO for the same handle and then relocated them in a CS,
787 * we would hit a deadlock in the kernel.
788 *
789 * The list of pairs is guarded by a mutex, of course. */
790 pipe_mutex_lock(mgr->bo_handles_mutex);
791
792 /* First check if there already is an existing bo for the handle. */
793 bo = util_hash_table_get(mgr->bo_handles, (void*)(uintptr_t)whandle->handle);
794 if (bo) {
795 /* Increase the refcount. */
796 struct pb_buffer *b = NULL;
797 pb_reference(&b, &bo->base);
798 goto done;
799 }
800
801 /* There isn't, create a new one. */
802 bo = CALLOC_STRUCT(radeon_bo);
803 if (!bo) {
804 goto fail;
805 }
806
807 /* Open the BO. */
808 open_arg.name = whandle->handle;
809 if (drmIoctl(ws->fd, DRM_IOCTL_GEM_OPEN, &open_arg)) {
810 FREE(bo);
811 goto fail;
812 }
813 bo->handle = open_arg.handle;
814 bo->name = whandle->handle;
815
816 /* Initialize it. */
817 pipe_reference_init(&bo->base.reference, 1);
818 bo->base.alignment = 0;
819 bo->base.usage = PB_USAGE_GPU_WRITE | PB_USAGE_GPU_READ;
820 bo->base.size = open_arg.size;
821 bo->base.vtbl = &radeon_bo_vtbl;
822 bo->mgr = mgr;
823 bo->rws = mgr->rws;
824 bo->va = 0;
825 pipe_mutex_init(bo->map_mutex);
826
827 util_hash_table_set(mgr->bo_handles, (void*)(uintptr_t)whandle->handle, bo);
828
829 done:
830 pipe_mutex_unlock(mgr->bo_handles_mutex);
831
832 if (stride)
833 *stride = whandle->stride;
834
835 if (mgr->va) {
836 struct drm_radeon_gem_va va;
837
838 bo->va_size = ((bo->base.size + 4095) & ~4095);
839 bo->va = radeon_bomgr_find_va(mgr, bo->va_size, 1 << 20);
840
841 va.handle = bo->handle;
842 va.operation = RADEON_VA_MAP;
843 va.vm_id = 0;
844 va.offset = bo->va;
845 va.flags = RADEON_VM_PAGE_READABLE |
846 RADEON_VM_PAGE_WRITEABLE |
847 RADEON_VM_PAGE_SNOOPED;
848 va.offset = bo->va;
849 r = drmCommandWriteRead(ws->fd, DRM_RADEON_GEM_VA, &va, sizeof(va));
850 if (r && va.operation == RADEON_VA_RESULT_ERROR) {
851 fprintf(stderr, "radeon: Failed to assign virtual address space\n");
852 radeon_bo_destroy(&bo->base);
853 return NULL;
854 }
855 if (va.operation == RADEON_VA_RESULT_VA_EXIST) {
856 radeon_bomgr_free_va(mgr, bo->va, bo->va_size);
857 bo->va = va.offset;
858 radeon_bomgr_force_va(mgr, bo->va, bo->va_size);
859 }
860 }
861
862 return (struct pb_buffer*)bo;
863
864 fail:
865 pipe_mutex_unlock(mgr->bo_handles_mutex);
866 return NULL;
867 }
868
869 static boolean radeon_winsys_bo_get_handle(struct pb_buffer *buffer,
870 unsigned stride,
871 struct winsys_handle *whandle)
872 {
873 struct drm_gem_flink flink;
874 struct radeon_bo *bo = get_radeon_bo(buffer);
875
876 memset(&flink, 0, sizeof(flink));
877
878 if (whandle->type == DRM_API_HANDLE_TYPE_SHARED) {
879 if (!bo->flinked) {
880 flink.handle = bo->handle;
881
882 if (ioctl(bo->rws->fd, DRM_IOCTL_GEM_FLINK, &flink)) {
883 return FALSE;
884 }
885
886 bo->flinked = TRUE;
887 bo->flink = flink.name;
888 }
889 whandle->handle = bo->flink;
890 } else if (whandle->type == DRM_API_HANDLE_TYPE_KMS) {
891 whandle->handle = bo->handle;
892 }
893
894 whandle->stride = stride;
895 return TRUE;
896 }
897
898 static uint64_t radeon_winsys_bo_va(struct pb_buffer *buffer)
899 {
900 struct radeon_bo *bo = get_radeon_bo(buffer);
901
902 return bo->va;
903 }
904
905 void radeon_bomgr_init_functions(struct radeon_drm_winsys *ws)
906 {
907 ws->base.buffer_get_cs_handle = radeon_drm_get_cs_handle;
908 ws->base.buffer_set_tiling = radeon_bo_set_tiling;
909 ws->base.buffer_get_tiling = radeon_bo_get_tiling;
910 ws->base.buffer_map = radeon_bo_map;
911 ws->base.buffer_unmap = pb_unmap;
912 ws->base.buffer_wait = radeon_bo_wait;
913 ws->base.buffer_is_busy = radeon_bo_is_busy;
914 ws->base.buffer_create = radeon_winsys_bo_create;
915 ws->base.buffer_from_handle = radeon_winsys_bo_from_handle;
916 ws->base.buffer_get_handle = radeon_winsys_bo_get_handle;
917 ws->base.buffer_get_virtual_address = radeon_winsys_bo_va;
918 }