mesa: add storage flags parameter to Driver.BufferData
[mesa.git] / src / mesa / drivers / dri / i965 / intel_buffer_objects.c
1 /**************************************************************************
2 *
3 * Copyright 2003 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 /**
29 * @file intel_buffer_objects.c
30 *
31 * This provides core GL buffer object functionality.
32 */
33
34 #include "main/imports.h"
35 #include "main/mtypes.h"
36 #include "main/macros.h"
37 #include "main/bufferobj.h"
38
39 #include "brw_context.h"
40 #include "intel_blit.h"
41 #include "intel_buffer_objects.h"
42 #include "intel_batchbuffer.h"
43
44 /**
45 * Map a buffer object; issue performance warnings if mapping causes stalls.
46 *
47 * This matches the drm_intel_bo_map API, but takes an additional human-readable
48 * name for the buffer object to use in the performance debug message.
49 */
50 int
51 brw_bo_map(struct brw_context *brw,
52 drm_intel_bo *bo, int write_enable,
53 const char *bo_name)
54 {
55 if (likely(!brw->perf_debug) || !drm_intel_bo_busy(bo))
56 return drm_intel_bo_map(bo, write_enable);
57
58 double start_time = get_time();
59
60 int ret = drm_intel_bo_map(bo, write_enable);
61
62 perf_debug("CPU mapping a busy %s BO stalled and took %.03f ms.\n",
63 bo_name, (get_time() - start_time) * 1000);
64
65 return ret;
66 }
67
68 int
69 brw_bo_map_gtt(struct brw_context *brw, drm_intel_bo *bo, const char *bo_name)
70 {
71 if (likely(!brw->perf_debug) || !drm_intel_bo_busy(bo))
72 return drm_intel_gem_bo_map_gtt(bo);
73
74 double start_time = get_time();
75
76 int ret = drm_intel_gem_bo_map_gtt(bo);
77
78 perf_debug("GTT mapping a busy %s BO stalled and took %.03f ms.\n",
79 bo_name, (get_time() - start_time) * 1000);
80
81 return ret;
82 }
83
84 static GLboolean
85 intel_bufferobj_unmap(struct gl_context * ctx, struct gl_buffer_object *obj);
86
87 static void
88 intel_bufferobj_mark_gpu_usage(struct intel_buffer_object *intel_obj,
89 uint32_t offset, uint32_t size)
90 {
91 intel_obj->gpu_active_start = MIN2(intel_obj->gpu_active_start, offset);
92 intel_obj->gpu_active_end = MAX2(intel_obj->gpu_active_end, offset + size);
93 }
94
95 static void
96 intel_bufferobj_mark_inactive(struct intel_buffer_object *intel_obj)
97 {
98 intel_obj->gpu_active_start = ~0;
99 intel_obj->gpu_active_end = 0;
100 }
101
102 /** Allocates a new drm_intel_bo to store the data for the buffer object. */
103 static void
104 intel_bufferobj_alloc_buffer(struct brw_context *brw,
105 struct intel_buffer_object *intel_obj)
106 {
107 intel_obj->buffer = drm_intel_bo_alloc(brw->bufmgr, "bufferobj",
108 intel_obj->Base.Size, 64);
109
110 /* the buffer might be bound as a uniform buffer, need to update it
111 */
112 brw->state.dirty.brw |= BRW_NEW_UNIFORM_BUFFER;
113
114 intel_bufferobj_mark_inactive(intel_obj);
115 }
116
117 static void
118 release_buffer(struct intel_buffer_object *intel_obj)
119 {
120 drm_intel_bo_unreference(intel_obj->buffer);
121 intel_obj->buffer = NULL;
122 }
123
124 /**
125 * The NewBufferObject() driver hook.
126 *
127 * Allocates a new intel_buffer_object structure and initializes it.
128 *
129 * There is some duplication between mesa's bufferobjects and our
130 * bufmgr buffers. Both have an integer handle and a hashtable to
131 * lookup an opaque structure. It would be nice if the handles and
132 * internal structure where somehow shared.
133 */
134 static struct gl_buffer_object *
135 intel_bufferobj_alloc(struct gl_context * ctx, GLuint name, GLenum target)
136 {
137 struct intel_buffer_object *obj = CALLOC_STRUCT(intel_buffer_object);
138
139 _mesa_initialize_buffer_object(ctx, &obj->Base, name, target);
140
141 obj->buffer = NULL;
142
143 return &obj->Base;
144 }
145
146 /**
147 * The DeleteBuffer() driver hook.
148 *
149 * Deletes a single OpenGL buffer object. Used by glDeleteBuffers().
150 */
151 static void
152 intel_bufferobj_free(struct gl_context * ctx, struct gl_buffer_object *obj)
153 {
154 struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
155
156 assert(intel_obj);
157
158 /* Buffer objects are automatically unmapped when deleting according
159 * to the spec, but Mesa doesn't do UnmapBuffer for us at context destroy
160 * (though it does if you call glDeleteBuffers)
161 */
162 if (obj->Pointer)
163 intel_bufferobj_unmap(ctx, obj);
164
165 drm_intel_bo_unreference(intel_obj->buffer);
166 free(intel_obj);
167 }
168
169
170 /**
171 * The BufferData() driver hook.
172 *
173 * Implements glBufferData(), which recreates a buffer object's data store
174 * and populates it with the given data, if present.
175 *
176 * Any data that was previously stored in the buffer object is lost.
177 *
178 * \return true for success, false if out of memory
179 */
180 static GLboolean
181 intel_bufferobj_data(struct gl_context * ctx,
182 GLenum target,
183 GLsizeiptrARB size,
184 const GLvoid * data,
185 GLenum usage,
186 GLbitfield storageFlags,
187 struct gl_buffer_object *obj)
188 {
189 struct brw_context *brw = brw_context(ctx);
190 struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
191
192 /* Part of the ABI, but this function doesn't use it.
193 */
194 (void) target;
195
196 intel_obj->Base.Size = size;
197 intel_obj->Base.Usage = usage;
198 intel_obj->Base.StorageFlags = storageFlags;
199
200 assert(!obj->Pointer); /* Mesa should have unmapped it */
201
202 if (intel_obj->buffer != NULL)
203 release_buffer(intel_obj);
204
205 if (size != 0) {
206 intel_bufferobj_alloc_buffer(brw, intel_obj);
207 if (!intel_obj->buffer)
208 return false;
209
210 if (data != NULL)
211 drm_intel_bo_subdata(intel_obj->buffer, 0, size, data);
212 }
213
214 return true;
215 }
216
217
218 /**
219 * The BufferSubData() driver hook.
220 *
221 * Implements glBufferSubData(), which replaces a portion of the data in a
222 * buffer object.
223 *
224 * If the data range specified by (size + offset) extends beyond the end of
225 * the buffer or if data is NULL, no copy is performed.
226 */
227 static void
228 intel_bufferobj_subdata(struct gl_context * ctx,
229 GLintptrARB offset,
230 GLsizeiptrARB size,
231 const GLvoid * data, struct gl_buffer_object *obj)
232 {
233 struct brw_context *brw = brw_context(ctx);
234 struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
235 bool busy;
236
237 if (size == 0)
238 return;
239
240 assert(intel_obj);
241
242 /* See if we can unsynchronized write the data into the user's BO. This
243 * avoids GPU stalls in unfortunately common user patterns (uploading
244 * sequentially into a BO, with draw calls in between each upload).
245 *
246 * Once we've hit this path, we mark this GL BO as preferring stalling to
247 * blits, so that we can hopefully hit this path again in the future
248 * (otherwise, an app that might occasionally stall but mostly not will end
249 * up with blitting all the time, at the cost of bandwidth)
250 */
251 if (brw->has_llc) {
252 if (offset + size <= intel_obj->gpu_active_start ||
253 intel_obj->gpu_active_end <= offset) {
254 drm_intel_gem_bo_map_unsynchronized(intel_obj->buffer);
255 memcpy(intel_obj->buffer->virtual + offset, data, size);
256 drm_intel_bo_unmap(intel_obj->buffer);
257
258 if (intel_obj->gpu_active_end > intel_obj->gpu_active_start)
259 intel_obj->prefer_stall_to_blit = true;
260 return;
261 }
262 }
263
264 busy =
265 drm_intel_bo_busy(intel_obj->buffer) ||
266 drm_intel_bo_references(brw->batch.bo, intel_obj->buffer);
267
268 if (busy) {
269 if (size == intel_obj->Base.Size) {
270 /* Replace the current busy bo so the subdata doesn't stall. */
271 drm_intel_bo_unreference(intel_obj->buffer);
272 intel_bufferobj_alloc_buffer(brw, intel_obj);
273 } else if (!intel_obj->prefer_stall_to_blit) {
274 perf_debug("Using a blit copy to avoid stalling on "
275 "glBufferSubData(%ld, %ld) (%ldkb) to a busy "
276 "(%d-%d) buffer object.\n",
277 (long)offset, (long)offset + size, (long)(size/1024),
278 intel_obj->gpu_active_start,
279 intel_obj->gpu_active_end);
280 drm_intel_bo *temp_bo =
281 drm_intel_bo_alloc(brw->bufmgr, "subdata temp", size, 64);
282
283 drm_intel_bo_subdata(temp_bo, 0, size, data);
284
285 intel_emit_linear_blit(brw,
286 intel_obj->buffer, offset,
287 temp_bo, 0,
288 size);
289
290 drm_intel_bo_unreference(temp_bo);
291 return;
292 } else {
293 perf_debug("Stalling on glBufferSubData(%ld, %ld) (%ldkb) to a busy "
294 "(%d-%d) buffer object. Use glMapBufferRange() to "
295 "avoid this.\n",
296 (long)offset, (long)offset + size, (long)(size/1024),
297 intel_obj->gpu_active_start,
298 intel_obj->gpu_active_end);
299 intel_batchbuffer_flush(brw);
300 }
301 }
302
303 drm_intel_bo_subdata(intel_obj->buffer, offset, size, data);
304 intel_bufferobj_mark_inactive(intel_obj);
305 }
306
307
308 /**
309 * The GetBufferSubData() driver hook.
310 *
311 * Implements glGetBufferSubData(), which copies a subrange of a buffer
312 * object into user memory.
313 */
314 static void
315 intel_bufferobj_get_subdata(struct gl_context * ctx,
316 GLintptrARB offset,
317 GLsizeiptrARB size,
318 GLvoid * data, struct gl_buffer_object *obj)
319 {
320 struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
321 struct brw_context *brw = brw_context(ctx);
322
323 assert(intel_obj);
324 if (drm_intel_bo_references(brw->batch.bo, intel_obj->buffer)) {
325 intel_batchbuffer_flush(brw);
326 }
327 drm_intel_bo_get_subdata(intel_obj->buffer, offset, size, data);
328
329 intel_bufferobj_mark_inactive(intel_obj);
330 }
331
332
333 /**
334 * The MapBufferRange() driver hook.
335 *
336 * This implements both glMapBufferRange() and glMapBuffer().
337 *
338 * The goal of this extension is to allow apps to accumulate their rendering
339 * at the same time as they accumulate their buffer object. Without it,
340 * you'd end up blocking on execution of rendering every time you mapped
341 * the buffer to put new data in.
342 *
343 * We support it in 3 ways: If unsynchronized, then don't bother
344 * flushing the batchbuffer before mapping the buffer, which can save blocking
345 * in many cases. If we would still block, and they allow the whole buffer
346 * to be invalidated, then just allocate a new buffer to replace the old one.
347 * If not, and we'd block, and they allow the subrange of the buffer to be
348 * invalidated, then we can make a new little BO, let them write into that,
349 * and blit it into the real BO at unmap time.
350 */
351 static void *
352 intel_bufferobj_map_range(struct gl_context * ctx,
353 GLintptr offset, GLsizeiptr length,
354 GLbitfield access, struct gl_buffer_object *obj)
355 {
356 struct brw_context *brw = brw_context(ctx);
357 struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
358
359 assert(intel_obj);
360
361 /* _mesa_MapBufferRange (GL entrypoint) sets these, but the vbo module also
362 * internally uses our functions directly.
363 */
364 obj->Offset = offset;
365 obj->Length = length;
366 obj->AccessFlags = access;
367
368 if (intel_obj->buffer == NULL) {
369 obj->Pointer = NULL;
370 return NULL;
371 }
372
373 /* If the access is synchronized (like a normal buffer mapping), then get
374 * things flushed out so the later mapping syncs appropriately through GEM.
375 * If the user doesn't care about existing buffer contents and mapping would
376 * cause us to block, then throw out the old buffer.
377 *
378 * If they set INVALIDATE_BUFFER, we can pitch the current contents to
379 * achieve the required synchronization.
380 */
381 if (!(access & GL_MAP_UNSYNCHRONIZED_BIT)) {
382 if (drm_intel_bo_references(brw->batch.bo, intel_obj->buffer)) {
383 if (access & GL_MAP_INVALIDATE_BUFFER_BIT) {
384 drm_intel_bo_unreference(intel_obj->buffer);
385 intel_bufferobj_alloc_buffer(brw, intel_obj);
386 } else {
387 perf_debug("Stalling on the GPU for mapping a busy buffer "
388 "object\n");
389 intel_batchbuffer_flush(brw);
390 }
391 } else if (drm_intel_bo_busy(intel_obj->buffer) &&
392 (access & GL_MAP_INVALIDATE_BUFFER_BIT)) {
393 drm_intel_bo_unreference(intel_obj->buffer);
394 intel_bufferobj_alloc_buffer(brw, intel_obj);
395 }
396 }
397
398 /* If the user is mapping a range of an active buffer object but
399 * doesn't require the current contents of that range, make a new
400 * BO, and we'll copy what they put in there out at unmap or
401 * FlushRange time.
402 */
403 if (!(access & GL_MAP_UNSYNCHRONIZED_BIT) &&
404 (access & GL_MAP_INVALIDATE_RANGE_BIT) &&
405 drm_intel_bo_busy(intel_obj->buffer)) {
406 /* Ensure that the base alignment of the allocation meets the alignment
407 * guarantees the driver has advertised to the application.
408 */
409 const unsigned alignment = ctx->Const.MinMapBufferAlignment;
410 const unsigned extra = (uintptr_t) offset % alignment;
411
412 if (access & GL_MAP_FLUSH_EXPLICIT_BIT) {
413 intel_obj->range_map_buffer = _mesa_align_malloc(length + extra,
414 alignment);
415 obj->Pointer = intel_obj->range_map_buffer + extra;
416 } else {
417 intel_obj->range_map_bo = drm_intel_bo_alloc(brw->bufmgr,
418 "range map",
419 length + extra,
420 alignment);
421 if (!(access & GL_MAP_READ_BIT)) {
422 drm_intel_gem_bo_map_gtt(intel_obj->range_map_bo);
423 } else {
424 drm_intel_bo_map(intel_obj->range_map_bo,
425 (access & GL_MAP_WRITE_BIT) != 0);
426 }
427 obj->Pointer = intel_obj->range_map_bo->virtual + extra;
428 }
429 return obj->Pointer;
430 }
431
432 if (access & GL_MAP_UNSYNCHRONIZED_BIT)
433 drm_intel_gem_bo_map_unsynchronized(intel_obj->buffer);
434 else if (!(access & GL_MAP_READ_BIT)) {
435 drm_intel_gem_bo_map_gtt(intel_obj->buffer);
436 intel_bufferobj_mark_inactive(intel_obj);
437 } else {
438 drm_intel_bo_map(intel_obj->buffer, (access & GL_MAP_WRITE_BIT) != 0);
439 intel_bufferobj_mark_inactive(intel_obj);
440 }
441
442 obj->Pointer = intel_obj->buffer->virtual + offset;
443 return obj->Pointer;
444 }
445
446 /**
447 * The FlushMappedBufferRange() driver hook.
448 *
449 * Implements glFlushMappedBufferRange(), which signifies that modifications
450 * have been made to a range of a mapped buffer, and it should be flushed.
451 *
452 * This is only used for buffers mapped with GL_MAP_FLUSH_EXPLICIT_BIT.
453 *
454 * Ideally we'd use a BO to avoid taking up cache space for the temporary
455 * data, but FlushMappedBufferRange may be followed by further writes to
456 * the pointer, so we would have to re-map after emitting our blit, which
457 * would defeat the point.
458 */
459 static void
460 intel_bufferobj_flush_mapped_range(struct gl_context *ctx,
461 GLintptr offset, GLsizeiptr length,
462 struct gl_buffer_object *obj)
463 {
464 struct brw_context *brw = brw_context(ctx);
465 struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
466 drm_intel_bo *temp_bo;
467
468 /* Unless we're in the range map using a temporary system buffer,
469 * there's no work to do.
470 */
471 if (intel_obj->range_map_buffer == NULL)
472 return;
473
474 if (length == 0)
475 return;
476
477 temp_bo = drm_intel_bo_alloc(brw->bufmgr, "range map flush", length, 64);
478
479 /* Use obj->Pointer instead of intel_obj->range_map_buffer because the
480 * former points to the actual mapping while the latter may be offset to
481 * meet alignment guarantees.
482 */
483 drm_intel_bo_subdata(temp_bo, 0, length, obj->Pointer);
484
485 intel_emit_linear_blit(brw,
486 intel_obj->buffer, obj->Offset + offset,
487 temp_bo, 0,
488 length);
489 intel_bufferobj_mark_gpu_usage(intel_obj, obj->Offset + offset, length);
490
491 drm_intel_bo_unreference(temp_bo);
492 }
493
494
495 /**
496 * The UnmapBuffer() driver hook.
497 *
498 * Implements glUnmapBuffer().
499 */
500 static GLboolean
501 intel_bufferobj_unmap(struct gl_context * ctx, struct gl_buffer_object *obj)
502 {
503 struct brw_context *brw = brw_context(ctx);
504 struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
505
506 assert(intel_obj);
507 assert(obj->Pointer);
508 if (intel_obj->range_map_buffer != NULL) {
509 /* Since we've emitted some blits to buffers that will (likely) be used
510 * in rendering operations in other cache domains in this batch, emit a
511 * flush. Once again, we wish for a domain tracker in libdrm to cover
512 * usage inside of a batchbuffer.
513 */
514 intel_batchbuffer_emit_mi_flush(brw);
515 _mesa_align_free(intel_obj->range_map_buffer);
516 intel_obj->range_map_buffer = NULL;
517 } else if (intel_obj->range_map_bo != NULL) {
518 const unsigned extra = obj->Pointer - intel_obj->range_map_bo->virtual;
519
520 drm_intel_bo_unmap(intel_obj->range_map_bo);
521
522 intel_emit_linear_blit(brw,
523 intel_obj->buffer, obj->Offset,
524 intel_obj->range_map_bo, extra,
525 obj->Length);
526 intel_bufferobj_mark_gpu_usage(intel_obj, obj->Offset, obj->Length);
527
528 /* Since we've emitted some blits to buffers that will (likely) be used
529 * in rendering operations in other cache domains in this batch, emit a
530 * flush. Once again, we wish for a domain tracker in libdrm to cover
531 * usage inside of a batchbuffer.
532 */
533 intel_batchbuffer_emit_mi_flush(brw);
534
535 drm_intel_bo_unreference(intel_obj->range_map_bo);
536 intel_obj->range_map_bo = NULL;
537 } else if (intel_obj->buffer != NULL) {
538 drm_intel_bo_unmap(intel_obj->buffer);
539 }
540 obj->Pointer = NULL;
541 obj->Offset = 0;
542 obj->Length = 0;
543
544 return true;
545 }
546
547 /**
548 * Gets a pointer to the object's BO, and marks the given range as being used
549 * on the GPU.
550 *
551 * Anywhere that uses buffer objects in the pipeline should be using this to
552 * mark the range of the buffer that is being accessed by the pipeline.
553 */
554 drm_intel_bo *
555 intel_bufferobj_buffer(struct brw_context *brw,
556 struct intel_buffer_object *intel_obj,
557 uint32_t offset, uint32_t size)
558 {
559 /* This is needed so that things like transform feedback and texture buffer
560 * objects that need a BO but don't want to check that they exist for
561 * draw-time validation can just always get a BO from a GL buffer object.
562 */
563 if (intel_obj->buffer == NULL)
564 intel_bufferobj_alloc_buffer(brw, intel_obj);
565
566 intel_bufferobj_mark_gpu_usage(intel_obj, offset, size);
567
568 return intel_obj->buffer;
569 }
570
571 /**
572 * The CopyBufferSubData() driver hook.
573 *
574 * Implements glCopyBufferSubData(), which copies a portion of one buffer
575 * object's data to another. Independent source and destination offsets
576 * are allowed.
577 */
578 static void
579 intel_bufferobj_copy_subdata(struct gl_context *ctx,
580 struct gl_buffer_object *src,
581 struct gl_buffer_object *dst,
582 GLintptr read_offset, GLintptr write_offset,
583 GLsizeiptr size)
584 {
585 struct brw_context *brw = brw_context(ctx);
586 struct intel_buffer_object *intel_src = intel_buffer_object(src);
587 struct intel_buffer_object *intel_dst = intel_buffer_object(dst);
588 drm_intel_bo *src_bo, *dst_bo;
589
590 if (size == 0)
591 return;
592
593 dst_bo = intel_bufferobj_buffer(brw, intel_dst, write_offset, size);
594 src_bo = intel_bufferobj_buffer(brw, intel_src, read_offset, size);
595
596 intel_emit_linear_blit(brw,
597 dst_bo, write_offset,
598 src_bo, read_offset, size);
599
600 /* Since we've emitted some blits to buffers that will (likely) be used
601 * in rendering operations in other cache domains in this batch, emit a
602 * flush. Once again, we wish for a domain tracker in libdrm to cover
603 * usage inside of a batchbuffer.
604 */
605 intel_batchbuffer_emit_mi_flush(brw);
606 }
607
608 void
609 intelInitBufferObjectFuncs(struct dd_function_table *functions)
610 {
611 functions->NewBufferObject = intel_bufferobj_alloc;
612 functions->DeleteBuffer = intel_bufferobj_free;
613 functions->BufferData = intel_bufferobj_data;
614 functions->BufferSubData = intel_bufferobj_subdata;
615 functions->GetBufferSubData = intel_bufferobj_get_subdata;
616 functions->MapBufferRange = intel_bufferobj_map_range;
617 functions->FlushMappedBufferRange = intel_bufferobj_flush_mapped_range;
618 functions->UnmapBuffer = intel_bufferobj_unmap;
619 functions->CopyBufferSubData = intel_bufferobj_copy_subdata;
620 }