2 * Copyright © 2011 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include "intel_batchbuffer.h"
27 #include "intel_mipmap_tree.h"
28 #include "intel_fbo.h"
30 #include "brw_context.h"
31 #include "brw_state.h"
33 #include "blorp/blorp_genX_exec.h"
36 #include "gen4_blorp_exec.h"
39 #include "brw_blorp.h"
42 blorp_emit_dwords(struct blorp_batch
*batch
, unsigned n
)
44 assert(batch
->blorp
->driver_ctx
== batch
->driver_batch
);
45 struct brw_context
*brw
= batch
->driver_batch
;
47 intel_batchbuffer_begin(brw
, n
);
48 uint32_t *map
= brw
->batch
.map_next
;
49 brw
->batch
.map_next
+= n
;
50 intel_batchbuffer_advance(brw
);
55 blorp_emit_reloc(struct blorp_batch
*batch
,
56 void *location
, struct blorp_address address
, uint32_t delta
)
58 assert(batch
->blorp
->driver_ctx
== batch
->driver_batch
);
59 struct brw_context
*brw
= batch
->driver_batch
;
62 if (GEN_GEN
< 6 && brw_ptr_in_state_buffer(&brw
->batch
, location
)) {
63 offset
= (char *)location
- (char *)brw
->batch
.state
.map
;
64 return brw_state_reloc(&brw
->batch
, offset
,
65 address
.buffer
, address
.offset
+ delta
,
69 assert(!brw_ptr_in_state_buffer(&brw
->batch
, location
));
71 offset
= (char *)location
- (char *)brw
->batch
.batch
.map
;
72 return brw_batch_reloc(&brw
->batch
, offset
,
73 address
.buffer
, address
.offset
+ delta
,
78 blorp_surface_reloc(struct blorp_batch
*batch
, uint32_t ss_offset
,
79 struct blorp_address address
, uint32_t delta
)
81 assert(batch
->blorp
->driver_ctx
== batch
->driver_batch
);
82 struct brw_context
*brw
= batch
->driver_batch
;
83 struct brw_bo
*bo
= address
.buffer
;
86 brw_state_reloc(&brw
->batch
, ss_offset
, bo
, address
.offset
+ delta
,
89 void *reloc_ptr
= (void *)brw
->batch
.state
.map
+ ss_offset
;
91 *(uint64_t *)reloc_ptr
= reloc_val
;
93 *(uint32_t *)reloc_ptr
= reloc_val
;
98 blorp_get_surface_address(struct blorp_batch
*blorp_batch
,
99 struct blorp_address address
)
101 /* We'll let blorp_surface_reloc write the address. */
105 #if GEN_GEN >= 7 && GEN_GEN < 10
106 static struct blorp_address
107 blorp_get_surface_base_address(struct blorp_batch
*batch
)
109 assert(batch
->blorp
->driver_ctx
== batch
->driver_batch
);
110 struct brw_context
*brw
= batch
->driver_batch
;
111 return (struct blorp_address
) {
112 .buffer
= brw
->batch
.state
.bo
,
119 blorp_alloc_dynamic_state(struct blorp_batch
*batch
,
124 assert(batch
->blorp
->driver_ctx
== batch
->driver_batch
);
125 struct brw_context
*brw
= batch
->driver_batch
;
127 return brw_state_batch(brw
, size
, alignment
, offset
);
131 blorp_alloc_binding_table(struct blorp_batch
*batch
, unsigned num_entries
,
132 unsigned state_size
, unsigned state_alignment
,
133 uint32_t *bt_offset
, uint32_t *surface_offsets
,
136 assert(batch
->blorp
->driver_ctx
== batch
->driver_batch
);
137 struct brw_context
*brw
= batch
->driver_batch
;
139 uint32_t *bt_map
= brw_state_batch(brw
,
140 num_entries
* sizeof(uint32_t), 32,
143 for (unsigned i
= 0; i
< num_entries
; i
++) {
144 surface_maps
[i
] = brw_state_batch(brw
,
145 state_size
, state_alignment
,
146 &(surface_offsets
)[i
]);
147 bt_map
[i
] = surface_offsets
[i
];
152 blorp_alloc_vertex_buffer(struct blorp_batch
*batch
, uint32_t size
,
153 struct blorp_address
*addr
)
155 assert(batch
->blorp
->driver_ctx
== batch
->driver_batch
);
156 struct brw_context
*brw
= batch
->driver_batch
;
158 /* From the Skylake PRM, 3DSTATE_VERTEX_BUFFERS:
160 * "The VF cache needs to be invalidated before binding and then using
161 * Vertex Buffers that overlap with any previously bound Vertex Buffer
162 * (at a 64B granularity) since the last invalidation. A VF cache
163 * invalidate is performed by setting the "VF Cache Invalidation Enable"
164 * bit in PIPE_CONTROL."
166 * This restriction first appears in the Skylake PRM but the internal docs
167 * also list it as being an issue on Broadwell. In order to avoid this
168 * problem, we align all vertex buffer allocations to 64 bytes.
171 void *data
= brw_state_batch(brw
, size
, 64, &offset
);
173 *addr
= (struct blorp_address
) {
174 .buffer
= brw
->batch
.state
.bo
,
177 /* The VF cache designers apparently cut corners, and made the cache
178 * only consider the bottom 32 bits of memory addresses. If you happen
179 * to have two vertex buffers which get placed exactly 4 GiB apart and
180 * use them in back-to-back draw calls, you can get collisions. To work
181 * around this problem, we restrict vertex buffers to the low 32 bits of
184 .reloc_flags
= RELOC_32BIT
,
193 .mocs
= GEN7_MOCS_L3
,
201 * See vf_invalidate_for_vb_48b_transitions in genX_state_upload.c.
204 blorp_vf_invalidate_for_vb_48b_transitions(struct blorp_batch
*batch
,
205 const struct blorp_address
*addrs
,
208 #if GEN_GEN >= 8 && GEN_GEN < 11
209 struct brw_context
*brw
= batch
->driver_batch
;
210 bool need_invalidate
= false;
212 for (unsigned i
= 0; i
< num_vbs
; i
++) {
213 struct brw_bo
*bo
= addrs
[i
].buffer
;
215 bo
&& (bo
->kflags
& EXEC_OBJECT_PINNED
) ? bo
->gtt_offset
>> 32u : 0;
217 if (high_bits
!= brw
->vb
.last_bo_high_bits
[i
]) {
218 need_invalidate
= true;
219 brw
->vb
.last_bo_high_bits
[i
] = high_bits
;
223 if (need_invalidate
) {
224 brw_emit_pipe_control_flush(brw
, PIPE_CONTROL_VF_CACHE_INVALIDATE
| PIPE_CONTROL_CS_STALL
);
230 static struct blorp_address
231 blorp_get_workaround_page(struct blorp_batch
*batch
)
233 assert(batch
->blorp
->driver_ctx
== batch
->driver_batch
);
234 struct brw_context
*brw
= batch
->driver_batch
;
236 return (struct blorp_address
) {
237 .buffer
= brw
->workaround_bo
,
243 blorp_flush_range(UNUSED
struct blorp_batch
*batch
, UNUSED
void *start
,
246 /* All allocated states come from the batch which we will flush before we
247 * submit it. There's nothing for us to do here.
252 blorp_emit_urb_config(struct blorp_batch
*batch
,
253 unsigned vs_entry_size
,
254 MAYBE_UNUSED
unsigned sf_entry_size
)
256 assert(batch
->blorp
->driver_ctx
== batch
->driver_batch
);
257 struct brw_context
*brw
= batch
->driver_batch
;
260 if (brw
->urb
.vsize
>= vs_entry_size
)
263 gen7_upload_urb(brw
, vs_entry_size
, false, false);
265 gen6_upload_urb(brw
, vs_entry_size
, false, 0);
267 /* We calculate it now and emit later. */
268 brw_calculate_urb_fence(brw
, 0, vs_entry_size
, sf_entry_size
);
273 genX(blorp_exec
)(struct blorp_batch
*batch
,
274 const struct blorp_params
*params
)
276 assert(batch
->blorp
->driver_ctx
== batch
->driver_batch
);
277 struct brw_context
*brw
= batch
->driver_batch
;
278 struct gl_context
*ctx
= &brw
->ctx
;
279 bool check_aperture_failed_once
= false;
282 /* The PIPE_CONTROL command description says:
284 * "Whenever a Binding Table Index (BTI) used by a Render Taget Message
285 * points to a different RENDER_SURFACE_STATE, SW must issue a Render
286 * Target Cache Flush by enabling this bit. When render target flush
287 * is set due to new association of BTI, PS Scoreboard Stall bit must
288 * be set in this packet."
290 brw_emit_pipe_control_flush(brw
,
291 PIPE_CONTROL_RENDER_TARGET_FLUSH
|
292 PIPE_CONTROL_STALL_AT_SCOREBOARD
);
295 /* Flush the sampler and render caches. We definitely need to flush the
296 * sampler cache so that we get updated contents from the render cache for
297 * the glBlitFramebuffer() source. Also, we are sometimes warned in the
298 * docs to flush the cache between reinterpretations of the same surface
299 * data with different formats, which blorp does for stencil and depth
302 if (params
->src
.enabled
)
303 brw_cache_flush_for_read(brw
, params
->src
.addr
.buffer
);
304 if (params
->dst
.enabled
) {
305 brw_cache_flush_for_render(brw
, params
->dst
.addr
.buffer
,
306 params
->dst
.view
.format
,
307 params
->dst
.aux_usage
);
309 if (params
->depth
.enabled
)
310 brw_cache_flush_for_depth(brw
, params
->depth
.addr
.buffer
);
311 if (params
->stencil
.enabled
)
312 brw_cache_flush_for_depth(brw
, params
->stencil
.addr
.buffer
);
314 brw_select_pipeline(brw
, BRW_RENDER_PIPELINE
);
317 intel_batchbuffer_require_space(brw
, 1400);
318 brw_require_statebuffer_space(brw
, 600);
319 intel_batchbuffer_save_state(brw
);
320 check_aperture_failed_once
|= intel_batchbuffer_saved_state_is_empty(brw
);
321 brw
->batch
.no_wrap
= true;
324 /* Emit workaround flushes when we switch from drawing to blorping. */
325 brw_emit_post_sync_nonzero_flush(brw
);
328 brw_upload_state_base_address(brw
);
331 gen7_l3_state
.emit(brw
);
335 brw_emit_depth_stall_flushes(brw
);
339 gen8_write_pma_stall_bits(brw
, 0);
342 blorp_emit(batch
, GENX(3DSTATE_DRAWING_RECTANGLE
), rect
) {
343 rect
.ClippedDrawingRectangleXMax
= MAX2(params
->x1
, params
->x0
) - 1;
344 rect
.ClippedDrawingRectangleYMax
= MAX2(params
->y1
, params
->y0
) - 1;
347 blorp_exec(batch
, params
);
349 brw
->batch
.no_wrap
= false;
351 /* Check if the blorp op we just did would make our batch likely to fail to
352 * map all the BOs into the GPU at batch exec time later. If so, flush the
353 * batch and try again with nothing else in the batch.
355 if (!brw_batch_has_aperture_space(brw
, 0)) {
356 if (!check_aperture_failed_once
) {
357 check_aperture_failed_once
= true;
358 intel_batchbuffer_reset_to_saved(brw
);
359 intel_batchbuffer_flush(brw
);
362 int ret
= intel_batchbuffer_flush(brw
);
363 WARN_ONCE(ret
== -ENOSPC
,
364 "i965: blorp emit exceeded available aperture space\n");
368 if (unlikely(brw
->always_flush_batch
))
369 intel_batchbuffer_flush(brw
);
371 /* We've smashed all state compared to what the normal 3D pipeline
372 * rendering tracks for GL.
374 brw
->ctx
.NewDriverState
|= BRW_NEW_BLORP
;
375 brw
->no_depth_or_stencil
= !params
->depth
.enabled
&&
376 !params
->stencil
.enabled
;
377 brw
->ib
.index_size
= -1;
379 if (params
->dst
.enabled
) {
380 brw_render_cache_add_bo(brw
, params
->dst
.addr
.buffer
,
381 params
->dst
.view
.format
,
382 params
->dst
.aux_usage
);
384 if (params
->depth
.enabled
)
385 brw_depth_cache_add_bo(brw
, params
->depth
.addr
.buffer
);
386 if (params
->stencil
.enabled
)
387 brw_depth_cache_add_bo(brw
, params
->stencil
.addr
.buffer
);