projects / mesa.git / blob
? search:


95905a8776f750891f91b2ca9398233984016f39
[mesa.git] / src / gallium / drivers / nouveau / nouveau_buffer.c
1
2 #include "util/u_inlines.h"
3 #include "util/u_memory.h"
4 #include "util/u_math.h"
5 #include "util/u_surface.h"
6
7 #include "nouveau_screen.h"
8 #include "nouveau_context.h"
9 #include "nouveau_winsys.h"
10 #include "nouveau_fence.h"
11 #include "nouveau_buffer.h"
12 #include "nouveau_mm.h"
13
14 #define NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD 192
15
16 struct nouveau_transfer {
17 struct pipe_transfer base;
18
19 uint8_t *map;
20 struct nouveau_bo *bo;
21 struct nouveau_mm_allocation *mm;
22 uint32_t offset;
23 };
24
25 static INLINE struct nouveau_transfer *
26 nouveau_transfer(struct pipe_transfer *transfer)
27 {
28 return (struct nouveau_transfer *)transfer;
29 }
30
31 static INLINE boolean
32 nouveau_buffer_malloc(struct nv04_resource *buf)
33 {
34 if (!buf->data)
35 buf->data = align_malloc(buf->base.width0, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
36 return !!buf->data;
37 }
38
39 static INLINE boolean
40 nouveau_buffer_allocate(struct nouveau_screen *screen,
41 struct nv04_resource *buf, unsigned domain)
42 {
43 uint32_t size = buf->base.width0;
44
45 if (buf->base.bind & (PIPE_BIND_CONSTANT_BUFFER |
46 PIPE_BIND_COMPUTE_RESOURCE |
47 PIPE_BIND_SHADER_RESOURCE))
48 size = align(size, 0x100);
49
50 if (domain == NOUVEAU_BO_VRAM) {
51 buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
52 &buf->bo, &buf->offset);
53 if (!buf->bo)
54 return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
55 NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0);
56 } else
57 if (domain == NOUVEAU_BO_GART) {
58 buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
59 &buf->bo, &buf->offset);
60 if (!buf->bo)
61 return FALSE;
62 NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0);
63 } else {
64 assert(domain == 0);
65 if (!nouveau_buffer_malloc(buf))
66 return FALSE;
67 }
68 buf->domain = domain;
69 if (buf->bo)
70 buf->address = buf->bo->offset + buf->offset;
71
72 return TRUE;
73 }
74
75 static INLINE void
76 release_allocation(struct nouveau_mm_allocation **mm,
77 struct nouveau_fence *fence)
78 {
79 nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
80 (*mm) = NULL;
81 }
82
83 INLINE void
84 nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
85 {
86 nouveau_bo_ref(NULL, &buf->bo);
87
88 if (buf->mm)
89 release_allocation(&buf->mm, buf->fence);
90
91 if (buf->domain == NOUVEAU_BO_VRAM)
92 NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_vid, -(uint64_t)buf->base.width0);
93 if (buf->domain == NOUVEAU_BO_GART)
94 NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_sys, -(uint64_t)buf->base.width0);
95
96 buf->domain = 0;
97 }
98
99 static INLINE boolean
100 nouveau_buffer_reallocate(struct nouveau_screen *screen,
101 struct nv04_resource *buf, unsigned domain)
102 {
103 nouveau_buffer_release_gpu_storage(buf);
104
105 nouveau_fence_ref(NULL, &buf->fence);
106 nouveau_fence_ref(NULL, &buf->fence_wr);
107
108 buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
109
110 return nouveau_buffer_allocate(screen, buf, domain);
111 }
112
113 static void
114 nouveau_buffer_destroy(struct pipe_screen *pscreen,
115 struct pipe_resource *presource)
116 {
117 struct nv04_resource *res = nv04_resource(presource);
118
119 nouveau_buffer_release_gpu_storage(res);
120
121 if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
122 align_free(res->data);
123
124 nouveau_fence_ref(NULL, &res->fence);
125 nouveau_fence_ref(NULL, &res->fence_wr);
126
127 FREE(res);
128
129 NOUVEAU_DRV_STAT(nouveau_screen(pscreen), buf_obj_current_count, -1);
130 }
131
132 static uint8_t *
133 nouveau_transfer_staging(struct nouveau_context *nv,
134 struct nouveau_transfer *tx, boolean permit_pb)
135 {
136 const unsigned adj = tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK;
137 const unsigned size = align(tx->base.box.width, 4) + adj;
138
139 if (!nv->push_data)
140 permit_pb = FALSE;
141
142 if ((size <= NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD) && permit_pb) {
143 tx->map = align_malloc(size, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
144 if (tx->map)
145 tx->map += adj;
146 } else {
147 tx->mm =
148 nouveau_mm_allocate(nv->screen->mm_GART, size, &tx->bo, &tx->offset);
149 if (tx->bo) {
150 tx->offset += adj;
151 if (!nouveau_bo_map(tx->bo, 0, NULL))
152 tx->map = (uint8_t *)tx->bo->map + tx->offset;
153 }
154 }
155 return tx->map;
156 }
157
158 /* Maybe just migrate to GART right away if we actually need to do this. */
159 static boolean
160 nouveau_transfer_read(struct nouveau_context *nv, struct nouveau_transfer *tx)
161 {
162 struct nv04_resource *buf = nv04_resource(tx->base.resource);
163 const unsigned base = tx->base.box.x;
164 const unsigned size = tx->base.box.width;
165
166 NOUVEAU_DRV_STAT(nv->screen, buf_read_bytes_staging_vid, size);
167
168 nv->copy_data(nv, tx->bo, tx->offset, NOUVEAU_BO_GART,
169 buf->bo, buf->offset + base, buf->domain, size);
170
171 if (nouveau_bo_wait(tx->bo, NOUVEAU_BO_RD, nv->client))
172 return FALSE;
173
174 if (buf->data)
175 memcpy(buf->data + base, tx->map, size);
176
177 return TRUE;
178 }
179
180 static void
181 nouveau_transfer_write(struct nouveau_context *nv, struct nouveau_transfer *tx,
182 unsigned offset, unsigned size)
183 {
184 struct nv04_resource *buf = nv04_resource(tx->base.resource);
185 uint8_t *data = tx->map + offset;
186 const unsigned base = tx->base.box.x + offset;
187 const boolean can_cb = !((base | size) & 3);
188
189 if (buf->data)
190 memcpy(data, buf->data + base, size);
191 else
192 buf->status |= NOUVEAU_BUFFER_STATUS_DIRTY;
193
194 if (buf->domain == NOUVEAU_BO_VRAM)
195 NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_vid, size);
196 if (buf->domain == NOUVEAU_BO_GART)
197 NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_sys, size);
198
199 if (tx->bo)
200 nv->copy_data(nv, buf->bo, buf->offset + base, buf->domain,
201 tx->bo, tx->offset + offset, NOUVEAU_BO_GART, size);
202 else
203 if ((buf->base.bind & PIPE_BIND_CONSTANT_BUFFER) && nv->push_cb && can_cb)
204 nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
205 base, size / 4, (const uint32_t *)data);
206 else
207 nv->push_data(nv, buf->bo, buf->offset + base, buf->domain, size, data);
208
209 nouveau_fence_ref(nv->screen->fence.current, &buf->fence);
210 nouveau_fence_ref(nv->screen->fence.current, &buf->fence_wr);
211 }
212
213
214 static INLINE boolean
215 nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
216 {
217 if (rw == PIPE_TRANSFER_READ) {
218 if (!buf->fence_wr)
219 return TRUE;
220 NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
221 !nouveau_fence_signalled(buf->fence_wr));
222 if (!nouveau_fence_wait(buf->fence_wr))
223 return FALSE;
224 } else {
225 if (!buf->fence)
226 return TRUE;
227 NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
228 !nouveau_fence_signalled(buf->fence));
229 if (!nouveau_fence_wait(buf->fence))
230 return FALSE;
231
232 nouveau_fence_ref(NULL, &buf->fence);
233 }
234 nouveau_fence_ref(NULL, &buf->fence_wr);
235
236 return TRUE;
237 }
238
239 static INLINE boolean
240 nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
241 {
242 if (rw == PIPE_TRANSFER_READ)
243 return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
244 else
245 return (buf->fence && !nouveau_fence_signalled(buf->fence));
246 }
247
248 static INLINE void
249 nouveau_buffer_transfer_init(struct nouveau_transfer *tx,
250 struct pipe_resource *resource,
251 const struct pipe_box *box,
252 unsigned usage)
253 {
254 tx->base.resource = resource;
255 tx->base.level = 0;
256 tx->base.usage = usage;
257 tx->base.box.x = box->x;
258 tx->base.box.y = 0;
259 tx->base.box.z = 0;
260 tx->base.box.width = box->width;
261 tx->base.box.height = 1;
262 tx->base.box.depth = 1;
263 tx->base.stride = 0;
264 tx->base.layer_stride = 0;
265
266 tx->bo = NULL;
267 tx->map = NULL;
268 }
269
270 static INLINE void
271 nouveau_buffer_transfer_del(struct nouveau_context *nv,
272 struct nouveau_transfer *tx)
273 {
274 if (tx->map) {
275 if (likely(tx->bo)) {
276 nouveau_bo_ref(NULL, &tx->bo);
277 if (tx->mm)
278 release_allocation(&tx->mm, nv->screen->fence.current);
279 } else {
280 align_free(tx->map -
281 (tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK));
282 }
283 }
284 }
285
286 static boolean
287 nouveau_buffer_cache(struct nouveau_context *nv, struct nv04_resource *buf)
288 {
289 struct nouveau_transfer tx;
290 boolean ret;
291 tx.base.resource = &buf->base;
292 tx.base.box.x = 0;
293 tx.base.box.width = buf->base.width0;
294 tx.bo = NULL;
295 tx.map = NULL;
296
297 if (!buf->data)
298 if (!nouveau_buffer_malloc(buf))
299 return FALSE;
300 if (!(buf->status & NOUVEAU_BUFFER_STATUS_DIRTY))
301 return TRUE;
302 nv->stats.buf_cache_count++;
303
304 if (!nouveau_transfer_staging(nv, &tx, FALSE))
305 return FALSE;
306
307 ret = nouveau_transfer_read(nv, &tx);
308 if (ret) {
309 buf->status &= ~NOUVEAU_BUFFER_STATUS_DIRTY;
310 memcpy(buf->data, tx.map, buf->base.width0);
311 }
312 nouveau_buffer_transfer_del(nv, &tx);
313 return ret;
314 }
315
316
317 #define NOUVEAU_TRANSFER_DISCARD \
318 (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
319
320 static INLINE boolean
321 nouveau_buffer_should_discard(struct nv04_resource *buf, unsigned usage)
322 {
323 if (!(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE))
324 return FALSE;
325 if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
326 return FALSE;
327 return buf->mm && nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE);
328 }
329
330 static void *
331 nouveau_buffer_transfer_map(struct pipe_context *pipe,
332 struct pipe_resource *resource,
333 unsigned level, unsigned usage,
334 const struct pipe_box *box,
335 struct pipe_transfer **ptransfer)
336 {
337 struct nouveau_context *nv = nouveau_context(pipe);
338 struct nv04_resource *buf = nv04_resource(resource);
339 struct nouveau_transfer *tx = MALLOC_STRUCT(nouveau_transfer);
340 uint8_t *map;
341 int ret;
342
343 if (!tx)
344 return NULL;
345 nouveau_buffer_transfer_init(tx, resource, box, usage);
346 *ptransfer = &tx->base;
347
348 if (usage & PIPE_TRANSFER_READ)
349 NOUVEAU_DRV_STAT(nv->screen, buf_transfers_rd, 1);
350 if (usage & PIPE_TRANSFER_WRITE)
351 NOUVEAU_DRV_STAT(nv->screen, buf_transfers_wr, 1);
352
353 if (buf->domain == NOUVEAU_BO_VRAM) {
354 if (usage & NOUVEAU_TRANSFER_DISCARD) {
355 if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
356 buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
357 nouveau_transfer_staging(nv, tx, TRUE);
358 } else {
359 if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
360 if (buf->data) {
361 align_free(buf->data);
362 buf->data = NULL;
363 }
364 nouveau_transfer_staging(nv, tx, FALSE);
365 nouveau_transfer_read(nv, tx);
366 } else {
367 if (usage & PIPE_TRANSFER_WRITE)
368 nouveau_transfer_staging(nv, tx, TRUE);
369 if (!buf->data)
370 nouveau_buffer_cache(nv, buf);
371 }
372 }
373 return buf->data ? (buf->data + box->x) : tx->map;
374 } else
375 if (unlikely(buf->domain == 0)) {
376 return buf->data + box->x;
377 }
378
379 if (nouveau_buffer_should_discard(buf, usage)) {
380 int ref = buf->base.reference.count - 1;
381 nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
382 if (ref > 0) /* any references inside context possible ? */
383 nv->invalidate_resource_storage(nv, &buf->base, ref);
384 }
385
386 ret = nouveau_bo_map(buf->bo,
387 buf->mm ? 0 : nouveau_screen_transfer_flags(usage),
388 nv->client);
389 if (ret) {
390 FREE(tx);
391 return NULL;
392 }
393 map = (uint8_t *)buf->bo->map + buf->offset + box->x;
394
395 /* using kernel fences only if !buf->mm */
396 if ((usage & PIPE_TRANSFER_UNSYNCHRONIZED) || !buf->mm)
397 return map;
398
399 if (nouveau_buffer_busy(buf, usage & PIPE_TRANSFER_READ_WRITE)) {
400 if (unlikely(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)) {
401 /* Discarding was not possible, must sync because
402 * subsequent transfers might use UNSYNCHRONIZED. */
403 nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
404 } else
405 if (usage & PIPE_TRANSFER_DISCARD_RANGE) {
406 nouveau_transfer_staging(nv, tx, TRUE);
407 map = tx->map;
408 } else
409 if (nouveau_buffer_busy(buf, PIPE_TRANSFER_READ)) {
410 if (usage & PIPE_TRANSFER_DONTBLOCK)
411 map = NULL;
412 else
413 nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
414 } else {
415 nouveau_transfer_staging(nv, tx, TRUE);
416 if (tx->map)
417 memcpy(tx->map, map, box->width);
418 map = tx->map;
419 }
420 }
421 if (!map)
422 FREE(tx);
423 return map;
424 }
425
426
427
428 static void
429 nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
430 struct pipe_transfer *transfer,
431 const struct pipe_box *box)
432 {
433 struct nouveau_transfer *tx = nouveau_transfer(transfer);
434 if (tx->map)
435 nouveau_transfer_write(nouveau_context(pipe), tx, box->x, box->width);
436 }
437
438 static void
439 nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
440 struct pipe_transfer *transfer)
441 {
442 struct nouveau_context *nv = nouveau_context(pipe);
443 struct nouveau_transfer *tx = nouveau_transfer(transfer);
444 struct nv04_resource *buf = nv04_resource(transfer->resource);
445
446 if (tx->base.usage & PIPE_TRANSFER_WRITE) {
447 if (!(tx->base.usage & PIPE_TRANSFER_FLUSH_EXPLICIT) && tx->map)
448 nouveau_transfer_write(nv, tx, 0, tx->base.box.width);
449
450 if (likely(buf->domain)) {
451 const uint8_t bind = buf->base.bind;
452 /* make sure we invalidate dedicated caches */
453 if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
454 nv->vbo_dirty = TRUE;
455 if (bind & (PIPE_BIND_CONSTANT_BUFFER))
456 nv->cb_dirty = TRUE;
457 }
458 }
459
460 if (!tx->bo && (tx->base.usage & PIPE_TRANSFER_WRITE))
461 NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_direct, tx->base.box.width);
462
463 nouveau_buffer_transfer_del(nv, tx);
464 FREE(tx);
465 }
466
467
468 void
469 nouveau_copy_buffer(struct nouveau_context *nv,
470 struct nv04_resource *dst, unsigned dstx,
471 struct nv04_resource *src, unsigned srcx, unsigned size)
472 {
473 assert(dst->base.target == PIPE_BUFFER && src->base.target == PIPE_BUFFER);
474
475 if (likely(dst->domain) && likely(src->domain)) {
476 nv->copy_data(nv,
477 dst->bo, dst->offset + dstx, dst->domain,
478 src->bo, src->offset + srcx, src->domain, size);
479
480 dst->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING;
481 nouveau_fence_ref(nv->screen->fence.current, &dst->fence);
482 nouveau_fence_ref(nv->screen->fence.current, &dst->fence_wr);
483
484 src->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
485 nouveau_fence_ref(nv->screen->fence.current, &src->fence);
486 } else {
487 struct pipe_box src_box;
488 src_box.x = srcx;
489 src_box.y = 0;
490 src_box.z = 0;
491 src_box.width = size;
492 src_box.height = 1;
493 src_box.depth = 1;
494 util_resource_copy_region(&nv->pipe,
495 &dst->base, 0, dstx, 0, 0,
496 &src->base, 0, &src_box);
497 }
498 }
499
500
501 void *
502 nouveau_resource_map_offset(struct nouveau_context *nv,
503 struct nv04_resource *res, uint32_t offset,
504 uint32_t flags)
505 {
506 if (unlikely(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
507 return res->data + offset;
508
509 if (res->domain == NOUVEAU_BO_VRAM) {
510 if (!res->data || (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
511 nouveau_buffer_cache(nv, res);
512 }
513 if (res->domain != NOUVEAU_BO_GART)
514 return res->data + offset;
515
516 if (res->mm) {
517 unsigned rw;
518 rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
519 nouveau_buffer_sync(res, rw);
520 if (nouveau_bo_map(res->bo, 0, NULL))
521 return NULL;
522 } else {
523 if (nouveau_bo_map(res->bo, flags, nv->client))
524 return NULL;
525 }
526 return (uint8_t *)res->bo->map + res->offset + offset;
527 }
528
529
530 const struct u_resource_vtbl nouveau_buffer_vtbl =
531 {
532 u_default_resource_get_handle, /* get_handle */
533 nouveau_buffer_destroy, /* resource_destroy */
534 nouveau_buffer_transfer_map, /* transfer_map */
535 nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
536 nouveau_buffer_transfer_unmap, /* transfer_unmap */
537 u_default_transfer_inline_write /* transfer_inline_write */
538 };
539
540 struct pipe_resource *
541 nouveau_buffer_create(struct pipe_screen *pscreen,
542 const struct pipe_resource *templ)
543 {
544 struct nouveau_screen *screen = nouveau_screen(pscreen);
545 struct nv04_resource *buffer;
546 boolean ret;
547
548 buffer = CALLOC_STRUCT(nv04_resource);
549 if (!buffer)
550 return NULL;
551
552 buffer->base = *templ;
553 buffer->vtbl = &nouveau_buffer_vtbl;
554 pipe_reference_init(&buffer->base.reference, 1);
555 buffer->base.screen = pscreen;
556
557 if (buffer->base.bind &
558 (screen->vidmem_bindings & screen->sysmem_bindings)) {
559 switch (buffer->base.usage) {
560 case PIPE_USAGE_DEFAULT:
561 case PIPE_USAGE_IMMUTABLE:
562 case PIPE_USAGE_STATIC:
563 buffer->domain = NOUVEAU_BO_VRAM;
564 break;
565 case PIPE_USAGE_DYNAMIC:
566 /* For most apps, we'd have to do staging transfers to avoid sync
567 * with this usage, and GART -> GART copies would be suboptimal.
568 */
569 buffer->domain = NOUVEAU_BO_VRAM;
570 break;
571 case PIPE_USAGE_STAGING:
572 case PIPE_USAGE_STREAM:
573 buffer->domain = NOUVEAU_BO_GART;
574 break;
575 default:
576 assert(0);
577 break;
578 }
579 } else {
580 if (buffer->base.bind & screen->vidmem_bindings)
581 buffer->domain = NOUVEAU_BO_VRAM;
582 else
583 if (buffer->base.bind & screen->sysmem_bindings)
584 buffer->domain = NOUVEAU_BO_GART;
585 }
586 ret = nouveau_buffer_allocate(screen, buffer, buffer->domain);
587
588 if (ret == FALSE)
589 goto fail;
590
591 if (buffer->domain == NOUVEAU_BO_VRAM && screen->hint_buf_keep_sysmem_copy)
592 nouveau_buffer_cache(NULL, buffer);
593
594 NOUVEAU_DRV_STAT(screen, buf_obj_current_count, 1);
595
596 return &buffer->base;
597
598 fail:
599 FREE(buffer);
600 return NULL;
601 }
602
603
604 struct pipe_resource *
605 nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
606 unsigned bytes, unsigned bind)
607 {
608 struct nv04_resource *buffer;
609
610 buffer = CALLOC_STRUCT(nv04_resource);
611 if (!buffer)
612 return NULL;
613
614 pipe_reference_init(&buffer->base.reference, 1);
615 buffer->vtbl = &nouveau_buffer_vtbl;
616 buffer->base.screen = pscreen;
617 buffer->base.format = PIPE_FORMAT_R8_UNORM;
618 buffer->base.usage = PIPE_USAGE_IMMUTABLE;
619 buffer->base.bind = bind;
620 buffer->base.width0 = bytes;
621 buffer->base.height0 = 1;
622 buffer->base.depth0 = 1;
623
624 buffer->data = ptr;
625 buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
626
627 return &buffer->base;
628 }
629
630 static INLINE boolean
631 nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
632 struct nouveau_bo *bo, unsigned offset, unsigned size)
633 {
634 if (!nouveau_buffer_malloc(buf))
635 return FALSE;
636 if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->client))
637 return FALSE;
638 memcpy(buf->data, (uint8_t *)bo->map + offset, size);
639 return TRUE;
640 }
641
642 /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
643 boolean
644 nouveau_buffer_migrate(struct nouveau_context *nv,
645 struct nv04_resource *buf, const unsigned new_domain)
646 {
647 struct nouveau_screen *screen = nv->screen;
648 struct nouveau_bo *bo;
649 const unsigned old_domain = buf->domain;
650 unsigned size = buf->base.width0;
651 unsigned offset;
652 int ret;
653
654 assert(new_domain != old_domain);
655
656 if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
657 if (!nouveau_buffer_allocate(screen, buf, new_domain))
658 return FALSE;
659 ret = nouveau_bo_map(buf->bo, 0, nv->client);
660 if (ret)
661 return ret;
662 memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
663 align_free(buf->data);
664 } else
665 if (old_domain != 0 && new_domain != 0) {
666 struct nouveau_mm_allocation *mm = buf->mm;
667
668 if (new_domain == NOUVEAU_BO_VRAM) {
669 /* keep a system memory copy of our data in case we hit a fallback */
670 if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
671 return FALSE;
672 if (nouveau_mesa_debug)
673 debug_printf("migrating %u KiB to VRAM\n", size / 1024);
674 }
675
676 offset = buf->offset;
677 bo = buf->bo;
678 buf->bo = NULL;
679 buf->mm = NULL;
680 nouveau_buffer_allocate(screen, buf, new_domain);
681
682 nv->copy_data(nv, buf->bo, buf->offset, new_domain,
683 bo, offset, old_domain, buf->base.width0);
684
685 nouveau_bo_ref(NULL, &bo);
686 if (mm)
687 release_allocation(&mm, screen->fence.current);
688 } else
689 if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
690 struct nouveau_transfer tx;
691 if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
692 return FALSE;
693 tx.base.resource = &buf->base;
694 tx.base.box.x = 0;
695 tx.base.box.width = buf->base.width0;
696 tx.bo = NULL;
697 tx.map = NULL;
698 if (!nouveau_transfer_staging(nv, &tx, FALSE))
699 return FALSE;
700 nouveau_transfer_write(nv, &tx, 0, tx.base.box.width);
701 nouveau_buffer_transfer_del(nv, &tx);
702 } else
703 return FALSE;
704
705 assert(buf->domain == new_domain);
706 return TRUE;
707 }
708
709 /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
710 * We'd like to only allocate @size bytes here, but then we'd have to rebase
711 * the vertex indices ...
712 */
713 boolean
714 nouveau_user_buffer_upload(struct nouveau_context *nv,
715 struct nv04_resource *buf,
716 unsigned base, unsigned size)
717 {
718 struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
719 int ret;
720
721 assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
722
723 buf->base.width0 = base + size;
724 if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
725 return FALSE;
726
727 ret = nouveau_bo_map(buf->bo, 0, nv->client);
728 if (ret)
729 return FALSE;
730 memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
731
732 return TRUE;
733 }
734
735
736 /* Scratch data allocation. */
737
738 static INLINE int
739 nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
740 unsigned size)
741 {
742 return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
743 4096, size, NULL, pbo);
744 }
745
746 void
747 nouveau_scratch_runout_release(struct nouveau_context *nv)
748 {
749 if (!nv->scratch.nr_runout)
750 return;
751 do {
752 --nv->scratch.nr_runout;
753 nouveau_bo_ref(NULL, &nv->scratch.runout[nv->scratch.nr_runout]);
754 } while (nv->scratch.nr_runout);
755
756 FREE(nv->scratch.runout);
757 nv->scratch.end = 0;
758 nv->scratch.runout = NULL;
759 }
760
761 /* Allocate an extra bo if we can't fit everything we need simultaneously.
762 * (Could happen for very large user arrays.)
763 */
764 static INLINE boolean
765 nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
766 {
767 int ret;
768 const unsigned n = nv->scratch.nr_runout++;
769
770 nv->scratch.runout = REALLOC(nv->scratch.runout,
771 (n + 0) * sizeof(*nv->scratch.runout),
772 (n + 1) * sizeof(*nv->scratch.runout));
773 nv->scratch.runout[n] = NULL;
774
775 ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout[n], size);
776 if (!ret) {
777 ret = nouveau_bo_map(nv->scratch.runout[n], 0, NULL);
778 if (ret)
779 nouveau_bo_ref(NULL, &nv->scratch.runout[--nv->scratch.nr_runout]);
780 }
781 if (!ret) {
782 nv->scratch.current = nv->scratch.runout[n];
783 nv->scratch.offset = 0;
784 nv->scratch.end = size;
785 nv->scratch.map = nv->scratch.current->map;
786 }
787 return !ret;
788 }
789
790 /* Continue to next scratch buffer, if available (no wrapping, large enough).
791 * Allocate it if it has not yet been created.
792 */
793 static INLINE boolean
794 nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
795 {
796 struct nouveau_bo *bo;
797 int ret;
798 const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
799
800 if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
801 return FALSE;
802 nv->scratch.id = i;
803
804 bo = nv->scratch.bo[i];
805 if (!bo) {
806 ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
807 if (ret)
808 return FALSE;
809 nv->scratch.bo[i] = bo;
810 }
811 nv->scratch.current = bo;
812 nv->scratch.offset = 0;
813 nv->scratch.end = nv->scratch.bo_size;
814
815 ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->client);
816 if (!ret)
817 nv->scratch.map = bo->map;
818 return !ret;
819 }
820
821 static boolean
822 nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
823 {
824 boolean ret;
825
826 ret = nouveau_scratch_next(nv, min_size);
827 if (!ret)
828 ret = nouveau_scratch_runout(nv, min_size);
829 return ret;
830 }
831
832
833 /* Copy data to a scratch buffer and return address & bo the data resides in. */
834 uint64_t
835 nouveau_scratch_data(struct nouveau_context *nv,
836 const void *data, unsigned base, unsigned size,
837 struct nouveau_bo **bo)
838 {
839 unsigned bgn = MAX2(base, nv->scratch.offset);
840 unsigned end = bgn + size;
841
842 if (end >= nv->scratch.end) {
843 end = base + size;
844 if (!nouveau_scratch_more(nv, end))
845 return 0;
846 bgn = base;
847 }
848 nv->scratch.offset = align(end, 4);
849
850 memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size);
851
852 *bo = nv->scratch.current;
853 return (*bo)->offset + (bgn - base);
854 }
855
856 void *
857 nouveau_scratch_get(struct nouveau_context *nv,
858 unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
859 {
860 unsigned bgn = nv->scratch.offset;
861 unsigned end = nv->scratch.offset + size;
862
863 if (end >= nv->scratch.end) {
864 end = size;
865 if (!nouveau_scratch_more(nv, end))
866 return NULL;
867 bgn = 0;
868 }
869 nv->scratch.offset = align(end, 4);
870
871 *pbo = nv->scratch.current;
872 *gpu_addr = nv->scratch.current->offset + bgn;
873 return nv->scratch.map + bgn;
874 }