projects / mesa.git / blob
? search:


aa40a54f43dabd68516819394e023225b5ee182c
[mesa.git] / src / gallium / drivers / radeon / r600_pipe_common.h
1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 *
23 * Authors: Marek Olšák <maraeo@gmail.com>
24 *
25 */
26
27 /**
28 * This file contains common screen and context structures and functions
29 * for r600g and radeonsi.
30 */
31
32 #ifndef R600_PIPE_COMMON_H
33 #define R600_PIPE_COMMON_H
34
35 #include <stdio.h>
36
37 #include "radeon/radeon_winsys.h"
38
39 #include "util/u_blitter.h"
40 #include "util/list.h"
41 #include "util/u_range.h"
42 #include "util/u_slab.h"
43 #include "util/u_suballoc.h"
44 #include "util/u_transfer.h"
45
46 #define ATI_VENDOR_ID 0x1002
47
48 #define R600_RESOURCE_FLAG_TRANSFER (PIPE_RESOURCE_FLAG_DRV_PRIV << 0)
49 #define R600_RESOURCE_FLAG_FLUSHED_DEPTH (PIPE_RESOURCE_FLAG_DRV_PRIV << 1)
50 #define R600_RESOURCE_FLAG_FORCE_TILING (PIPE_RESOURCE_FLAG_DRV_PRIV << 2)
51 #define R600_RESOURCE_FLAG_DISABLE_DCC (PIPE_RESOURCE_FLAG_DRV_PRIV << 3)
52
53 #define R600_CONTEXT_STREAMOUT_FLUSH (1u << 0)
54 /* Pipeline & streamout query controls. */
55 #define R600_CONTEXT_START_PIPELINE_STATS (1u << 1)
56 #define R600_CONTEXT_STOP_PIPELINE_STATS (1u << 2)
57 #define R600_CONTEXT_PRIVATE_FLAG (1u << 3)
58
59 /* special primitive types */
60 #define R600_PRIM_RECTANGLE_LIST PIPE_PRIM_MAX
61
62 /* Debug flags. */
63 /* logging */
64 #define DBG_TEX (1 << 0)
65 /* gap - reuse */
66 #define DBG_COMPUTE (1 << 2)
67 #define DBG_VM (1 << 3)
68 /* gap - reuse */
69 /* shader logging */
70 #define DBG_FS (1 << 5)
71 #define DBG_VS (1 << 6)
72 #define DBG_GS (1 << 7)
73 #define DBG_PS (1 << 8)
74 #define DBG_CS (1 << 9)
75 #define DBG_TCS (1 << 10)
76 #define DBG_TES (1 << 11)
77 #define DBG_NO_IR (1 << 12)
78 #define DBG_NO_TGSI (1 << 13)
79 #define DBG_NO_ASM (1 << 14)
80 #define DBG_PREOPT_IR (1 << 15)
81 /* gaps */
82 #define DBG_TEST_DMA (1 << 20)
83 /* Bits 21-31 are reserved for the r600g driver. */
84 /* features */
85 #define DBG_NO_ASYNC_DMA (1llu << 32)
86 #define DBG_NO_HYPERZ (1llu << 33)
87 #define DBG_NO_DISCARD_RANGE (1llu << 34)
88 #define DBG_NO_2D_TILING (1llu << 35)
89 #define DBG_NO_TILING (1llu << 36)
90 #define DBG_SWITCH_ON_EOP (1llu << 37)
91 #define DBG_FORCE_DMA (1llu << 38)
92 #define DBG_PRECOMPILE (1llu << 39)
93 #define DBG_INFO (1llu << 40)
94 #define DBG_NO_WC (1llu << 41)
95 #define DBG_CHECK_VM (1llu << 42)
96 #define DBG_NO_DCC (1llu << 43)
97 #define DBG_NO_DCC_CLEAR (1llu << 44)
98 #define DBG_NO_RB_PLUS (1llu << 45)
99 #define DBG_SI_SCHED (1llu << 46)
100 #define DBG_MONOLITHIC_SHADERS (1llu << 47)
101 #define DBG_NO_CE (1llu << 48)
102 #define DBG_UNSAFE_MATH (1llu << 49)
103 #define DBG_NO_DCC_FB (1llu << 50)
104
105 #define R600_MAP_BUFFER_ALIGNMENT 64
106 #define R600_MAX_VIEWPORTS 16
107
108 enum r600_coherency {
109 R600_COHERENCY_NONE, /* no cache flushes needed */
110 R600_COHERENCY_SHADER,
111 R600_COHERENCY_CB_META,
112 };
113
114 #ifdef PIPE_ARCH_BIG_ENDIAN
115 #define R600_BIG_ENDIAN 1
116 #else
117 #define R600_BIG_ENDIAN 0
118 #endif
119
120 struct r600_common_context;
121 struct r600_perfcounters;
122 struct tgsi_shader_info;
123
124 struct radeon_shader_reloc {
125 char name[32];
126 uint64_t offset;
127 };
128
129 struct radeon_shader_binary {
130 /** Shader code */
131 unsigned char *code;
132 unsigned code_size;
133
134 /** Config/Context register state that accompanies this shader.
135 * This is a stream of dword pairs. First dword contains the
136 * register address, the second dword contains the value.*/
137 unsigned char *config;
138 unsigned config_size;
139
140 /** The number of bytes of config information for each global symbol.
141 */
142 unsigned config_size_per_symbol;
143
144 /** Constant data accessed by the shader. This will be uploaded
145 * into a constant buffer. */
146 unsigned char *rodata;
147 unsigned rodata_size;
148
149 /** List of symbol offsets for the shader */
150 uint64_t *global_symbol_offsets;
151 unsigned global_symbol_count;
152
153 struct radeon_shader_reloc *relocs;
154 unsigned reloc_count;
155
156 /** Disassembled shader in a string. */
157 char *disasm_string;
158 char *llvm_ir_string;
159 };
160
161 void radeon_shader_binary_init(struct radeon_shader_binary *b);
162 void radeon_shader_binary_clean(struct radeon_shader_binary *b);
163
164 /* Only 32-bit buffer allocations are supported, gallium doesn't support more
165 * at the moment.
166 */
167 struct r600_resource {
168 struct u_resource b;
169
170 /* Winsys objects. */
171 struct pb_buffer *buf;
172 uint64_t gpu_address;
173 /* Memory usage if the buffer placement is optimal. */
174 uint64_t vram_usage;
175 uint64_t gart_usage;
176
177 /* Resource properties. */
178 uint64_t bo_size;
179 unsigned bo_alignment;
180 enum radeon_bo_domain domains;
181 enum radeon_bo_flag flags;
182
183 /* The buffer range which is initialized (with a write transfer,
184 * streamout, DMA, or as a random access target). The rest of
185 * the buffer is considered invalid and can be mapped unsynchronized.
186 *
187 * This allows unsychronized mapping of a buffer range which hasn't
188 * been used yet. It's for applications which forget to use
189 * the unsynchronized map flag and expect the driver to figure it out.
190 */
191 struct util_range valid_buffer_range;
192
193 /* For buffers only. This indicates that a write operation has been
194 * performed by TC L2, but the cache hasn't been flushed.
195 * Any hw block which doesn't use or bypasses TC L2 should check this
196 * flag and flush the cache before using the buffer.
197 *
198 * For example, TC L2 must be flushed if a buffer which has been
199 * modified by a shader store instruction is about to be used as
200 * an index buffer. The reason is that VGT DMA index fetching doesn't
201 * use TC L2.
202 */
203 bool TC_L2_dirty;
204
205 /* Whether the resource has been exported via resource_get_handle. */
206 bool is_shared;
207 unsigned external_usage; /* PIPE_HANDLE_USAGE_* */
208 };
209
210 struct r600_transfer {
211 struct pipe_transfer transfer;
212 struct r600_resource *staging;
213 unsigned offset;
214 };
215
216 struct r600_fmask_info {
217 uint64_t offset;
218 uint64_t size;
219 unsigned alignment;
220 unsigned pitch_in_pixels;
221 unsigned bank_height;
222 unsigned slice_tile_max;
223 unsigned tile_mode_index;
224 };
225
226 struct r600_cmask_info {
227 uint64_t offset;
228 uint64_t size;
229 unsigned alignment;
230 unsigned pitch;
231 unsigned height;
232 unsigned xalign;
233 unsigned yalign;
234 unsigned slice_tile_max;
235 unsigned base_address_reg;
236 };
237
238 struct r600_htile_info {
239 unsigned pitch;
240 unsigned height;
241 unsigned xalign;
242 unsigned yalign;
243 };
244
245 struct r600_texture {
246 struct r600_resource resource;
247
248 uint64_t size;
249 unsigned num_level0_transfers;
250 bool is_depth;
251 bool db_compatible;
252 bool can_sample_z;
253 bool can_sample_s;
254 unsigned dirty_level_mask; /* each bit says if that mipmap is compressed */
255 unsigned stencil_dirty_level_mask; /* each bit says if that mipmap is compressed */
256 struct r600_texture *flushed_depth_texture;
257 struct radeon_surf surface;
258
259 /* Colorbuffer compression and fast clear. */
260 struct r600_fmask_info fmask;
261 struct r600_cmask_info cmask;
262 struct r600_resource *cmask_buffer;
263 uint64_t dcc_offset; /* 0 = disabled */
264 unsigned cb_color_info; /* fast clear enable bit */
265 unsigned color_clear_value[2];
266 unsigned last_msaa_resolve_target_micro_mode;
267
268 /* Depth buffer compression and fast clear. */
269 struct r600_htile_info htile;
270 struct r600_resource *htile_buffer;
271 bool depth_cleared; /* if it was cleared at least once */
272 float depth_clear_value;
273 bool stencil_cleared; /* if it was cleared at least once */
274 uint8_t stencil_clear_value;
275
276 bool non_disp_tiling; /* R600-Cayman only */
277
278 /* Whether the texture is a displayable back buffer and needs DCC
279 * decompression, which is expensive. Therefore, it's enabled only
280 * if statistics suggest that it will pay off and it's allocated
281 * separately. It can't be bound as a sampler by apps. Limited to
282 * target == 2D and last_level == 0. If enabled, dcc_offset contains
283 * the absolute GPUVM address, not the relative one.
284 */
285 struct r600_resource *dcc_separate_buffer;
286 /* When DCC is temporarily disabled, the separate buffer is here. */
287 struct r600_resource *last_dcc_separate_buffer;
288 /* We need to track DCC dirtiness, because st/dri usually calls
289 * flush_resource twice per frame (not a bug) and we don't wanna
290 * decompress DCC twice. Also, the dirty tracking must be done even
291 * if DCC isn't used, because it's required by the DCC usage analysis
292 * for a possible future enablement.
293 */
294 bool separate_dcc_dirty;
295 /* Statistics gathering for the DCC enablement heuristic. */
296 bool dcc_gather_statistics;
297 /* Estimate of how much this color buffer is written to in units of
298 * full-screen draws: ps_invocations / (width * height)
299 * Shader kills, late Z, and blending with trivial discards make it
300 * inaccurate (we need to count CB updates, not PS invocations).
301 */
302 unsigned ps_draw_ratio;
303 /* The number of clears since the last DCC usage analysis. */
304 unsigned num_slow_clears;
305
306 /* Counter that should be non-zero if the texture is bound to a
307 * framebuffer. Implemented in radeonsi only.
308 */
309 uint32_t framebuffers_bound;
310 };
311
312 struct r600_surface {
313 struct pipe_surface base;
314 const struct radeon_surf_level *level_info;
315
316 bool color_initialized;
317 bool depth_initialized;
318
319 /* Misc. color flags. */
320 bool alphatest_bypass;
321 bool export_16bpc;
322 bool color_is_int8;
323
324 /* Color registers. */
325 unsigned cb_color_info;
326 unsigned cb_color_base;
327 unsigned cb_color_view;
328 unsigned cb_color_size; /* R600 only */
329 unsigned cb_color_dim; /* EG only */
330 unsigned cb_color_pitch; /* EG and later */
331 unsigned cb_color_slice; /* EG and later */
332 unsigned cb_color_attrib; /* EG and later */
333 unsigned cb_dcc_control; /* VI and later */
334 unsigned cb_color_fmask; /* CB_COLORn_FMASK (EG and later) or CB_COLORn_FRAG (r600) */
335 unsigned cb_color_fmask_slice; /* EG and later */
336 unsigned cb_color_cmask; /* CB_COLORn_TILE (r600 only) */
337 unsigned cb_color_mask; /* R600 only */
338 unsigned spi_shader_col_format; /* SI+, no blending, no alpha-to-coverage. */
339 unsigned spi_shader_col_format_alpha; /* SI+, alpha-to-coverage */
340 unsigned spi_shader_col_format_blend; /* SI+, blending without alpha. */
341 unsigned spi_shader_col_format_blend_alpha; /* SI+, blending with alpha. */
342 struct r600_resource *cb_buffer_fmask; /* Used for FMASK relocations. R600 only */
343 struct r600_resource *cb_buffer_cmask; /* Used for CMASK relocations. R600 only */
344
345 /* DB registers. */
346 unsigned db_depth_info; /* R600 only, then SI and later */
347 unsigned db_z_info; /* EG and later */
348 unsigned db_depth_base; /* DB_Z_READ/WRITE_BASE (EG and later) or DB_DEPTH_BASE (r600) */
349 unsigned db_depth_view;
350 unsigned db_depth_size;
351 unsigned db_depth_slice; /* EG and later */
352 unsigned db_stencil_base; /* EG and later */
353 unsigned db_stencil_info; /* EG and later */
354 unsigned db_prefetch_limit; /* R600 only */
355 unsigned db_htile_surface;
356 unsigned db_htile_data_base;
357 unsigned db_preload_control; /* EG and later */
358 };
359
360 struct r600_common_screen {
361 struct pipe_screen b;
362 struct radeon_winsys *ws;
363 enum radeon_family family;
364 enum chip_class chip_class;
365 struct radeon_info info;
366 uint64_t debug_flags;
367 bool has_cp_dma;
368 bool has_streamout;
369
370 /* Texture filter settings. */
371 int force_aniso; /* -1 = disabled */
372
373 /* Auxiliary context. Mainly used to initialize resources.
374 * It must be locked prior to using and flushed before unlocking. */
375 struct pipe_context *aux_context;
376 pipe_mutex aux_context_lock;
377
378 /* This must be in the screen, because UE4 uses one context for
379 * compilation and another one for rendering.
380 */
381 unsigned num_compilations;
382 /* Along with ST_DEBUG=precompile, this should show if applications
383 * are loading shaders on demand. This is a monotonic counter.
384 */
385 unsigned num_shaders_created;
386
387 /* GPU load thread. */
388 pipe_mutex gpu_load_mutex;
389 pipe_thread gpu_load_thread;
390 unsigned gpu_load_counter_busy;
391 unsigned gpu_load_counter_idle;
392 volatile unsigned gpu_load_stop_thread; /* bool */
393
394 char renderer_string[100];
395
396 /* Performance counters. */
397 struct r600_perfcounters *perfcounters;
398
399 /* If pipe_screen wants to re-emit the framebuffer state of all
400 * contexts, it should atomically increment this. Each context will
401 * compare this with its own last known value of the counter before
402 * drawing and re-emit the framebuffer state accordingly.
403 */
404 unsigned dirty_fb_counter;
405
406 /* Atomically increment this counter when an existing texture's
407 * metadata is enabled or disabled in a way that requires changing
408 * contexts' compressed texture binding masks.
409 */
410 unsigned compressed_colortex_counter;
411
412 /* Atomically increment this counter when an existing texture's
413 * backing buffer or tile mode parameters have changed that requires
414 * recomputation of shader descriptors.
415 */
416 unsigned dirty_tex_descriptor_counter;
417
418 void (*query_opaque_metadata)(struct r600_common_screen *rscreen,
419 struct r600_texture *rtex,
420 struct radeon_bo_metadata *md);
421
422 void (*apply_opaque_metadata)(struct r600_common_screen *rscreen,
423 struct r600_texture *rtex,
424 struct radeon_bo_metadata *md);
425 };
426
427 /* This encapsulates a state or an operation which can emitted into the GPU
428 * command stream. */
429 struct r600_atom {
430 void (*emit)(struct r600_common_context *ctx, struct r600_atom *state);
431 unsigned num_dw;
432 unsigned short id;
433 };
434
435 struct r600_so_target {
436 struct pipe_stream_output_target b;
437
438 /* The buffer where BUFFER_FILLED_SIZE is stored. */
439 struct r600_resource *buf_filled_size;
440 unsigned buf_filled_size_offset;
441 bool buf_filled_size_valid;
442
443 unsigned stride_in_dw;
444 };
445
446 struct r600_streamout {
447 struct r600_atom begin_atom;
448 bool begin_emitted;
449 unsigned num_dw_for_end;
450
451 unsigned enabled_mask;
452 unsigned num_targets;
453 struct r600_so_target *targets[PIPE_MAX_SO_BUFFERS];
454
455 unsigned append_bitmask;
456 bool suspended;
457
458 /* External state which comes from the vertex shader,
459 * it must be set explicitly when binding a shader. */
460 unsigned *stride_in_dw;
461 unsigned enabled_stream_buffers_mask; /* stream0 buffers0-3 in 4 LSB */
462
463 /* The state of VGT_STRMOUT_BUFFER_(CONFIG|EN). */
464 unsigned hw_enabled_mask;
465
466 /* The state of VGT_STRMOUT_(CONFIG|EN). */
467 struct r600_atom enable_atom;
468 bool streamout_enabled;
469 bool prims_gen_query_enabled;
470 int num_prims_gen_queries;
471 };
472
473 struct r600_signed_scissor {
474 int minx;
475 int miny;
476 int maxx;
477 int maxy;
478 };
479
480 struct r600_scissors {
481 struct r600_atom atom;
482 unsigned dirty_mask;
483 struct pipe_scissor_state states[R600_MAX_VIEWPORTS];
484 };
485
486 struct r600_viewports {
487 struct r600_atom atom;
488 unsigned dirty_mask;
489 unsigned depth_range_dirty_mask;
490 struct pipe_viewport_state states[R600_MAX_VIEWPORTS];
491 struct r600_signed_scissor as_scissor[R600_MAX_VIEWPORTS];
492 };
493
494 struct r600_ring {
495 struct radeon_winsys_cs *cs;
496 void (*flush)(void *ctx, unsigned flags,
497 struct pipe_fence_handle **fence);
498 };
499
500 /* Saved CS data for debugging features. */
501 struct radeon_saved_cs {
502 uint32_t *ib;
503 unsigned num_dw;
504
505 struct radeon_bo_list_item *bo_list;
506 unsigned bo_count;
507 };
508
509 struct r600_common_context {
510 struct pipe_context b; /* base class */
511
512 struct r600_common_screen *screen;
513 struct radeon_winsys *ws;
514 struct radeon_winsys_ctx *ctx;
515 enum radeon_family family;
516 enum chip_class chip_class;
517 struct r600_ring gfx;
518 struct r600_ring dma;
519 struct pipe_fence_handle *last_gfx_fence;
520 struct pipe_fence_handle *last_sdma_fence;
521 unsigned num_gfx_cs_flushes;
522 unsigned initial_gfx_cs_size;
523 unsigned gpu_reset_counter;
524 unsigned last_dirty_fb_counter;
525 unsigned last_compressed_colortex_counter;
526 unsigned last_dirty_tex_descriptor_counter;
527
528 struct u_upload_mgr *uploader;
529 struct u_suballocator *allocator_zeroed_memory;
530 struct util_slab_mempool pool_transfers;
531
532 /* Current unaccounted memory usage. */
533 uint64_t vram;
534 uint64_t gtt;
535
536 /* States. */
537 struct r600_streamout streamout;
538 struct r600_scissors scissors;
539 struct r600_viewports viewports;
540 bool scissor_enabled;
541 bool clip_halfz;
542 bool vs_writes_viewport_index;
543 bool vs_disables_clipping_viewport;
544
545 /* Additional context states. */
546 unsigned flags; /* flush flags */
547
548 /* Queries. */
549 /* Maintain the list of active queries for pausing between IBs. */
550 int num_occlusion_queries;
551 int num_perfect_occlusion_queries;
552 struct list_head active_queries;
553 unsigned num_cs_dw_queries_suspend;
554 /* Additional hardware info. */
555 unsigned backend_mask;
556 unsigned max_db; /* for OQ */
557 /* Misc stats. */
558 unsigned num_draw_calls;
559 unsigned num_spill_draw_calls;
560 unsigned num_compute_calls;
561 unsigned num_spill_compute_calls;
562 unsigned num_dma_calls;
563 unsigned num_vs_flushes;
564 unsigned num_ps_flushes;
565 unsigned num_cs_flushes;
566 uint64_t num_alloc_tex_transfer_bytes;
567 unsigned last_tex_ps_draw_ratio; /* for query */
568
569 /* Render condition. */
570 struct r600_atom render_cond_atom;
571 struct pipe_query *render_cond;
572 unsigned render_cond_mode;
573 bool render_cond_invert;
574 bool render_cond_force_off; /* for u_blitter */
575
576 /* MSAA sample locations.
577 * The first index is the sample index.
578 * The second index is the coordinate: X, Y. */
579 float sample_locations_1x[1][2];
580 float sample_locations_2x[2][2];
581 float sample_locations_4x[4][2];
582 float sample_locations_8x[8][2];
583 float sample_locations_16x[16][2];
584
585 /* Statistics gathering for the DCC enablement heuristic. It can't be
586 * in r600_texture because r600_texture can be shared by multiple
587 * contexts. This is for back buffers only. We shouldn't get too many
588 * of those.
589 *
590 * X11 DRI3 rotates among a finite set of back buffers. They should
591 * all fit in this array. If they don't, separate DCC might never be
592 * enabled by DCC stat gathering.
593 */
594 struct {
595 struct r600_texture *tex;
596 /* Query queue: 0 = usually active, 1 = waiting, 2 = readback. */
597 struct pipe_query *ps_stats[3];
598 /* If all slots are used and another slot is needed,
599 * the least recently used slot is evicted based on this. */
600 int64_t last_use_timestamp;
601 bool query_active;
602 } dcc_stats[5];
603
604 /* The list of all texture buffer objects in this context.
605 * This list is walked when a buffer is invalidated/reallocated and
606 * the GPU addresses are updated. */
607 struct list_head texture_buffers;
608
609 struct pipe_debug_callback debug;
610
611 /* Copy one resource to another using async DMA. */
612 void (*dma_copy)(struct pipe_context *ctx,
613 struct pipe_resource *dst,
614 unsigned dst_level,
615 unsigned dst_x, unsigned dst_y, unsigned dst_z,
616 struct pipe_resource *src,
617 unsigned src_level,
618 const struct pipe_box *src_box);
619
620 void (*clear_buffer)(struct pipe_context *ctx, struct pipe_resource *dst,
621 uint64_t offset, uint64_t size, unsigned value,
622 enum r600_coherency coher);
623
624 void (*blit_decompress_depth)(struct pipe_context *ctx,
625 struct r600_texture *texture,
626 struct r600_texture *staging,
627 unsigned first_level, unsigned last_level,
628 unsigned first_layer, unsigned last_layer,
629 unsigned first_sample, unsigned last_sample);
630
631 void (*decompress_dcc)(struct pipe_context *ctx,
632 struct r600_texture *rtex);
633
634 /* Reallocate the buffer and update all resource bindings where
635 * the buffer is bound, including all resource descriptors. */
636 void (*invalidate_buffer)(struct pipe_context *ctx, struct pipe_resource *buf);
637
638 /* Enable or disable occlusion queries. */
639 void (*set_occlusion_query_state)(struct pipe_context *ctx, bool enable);
640
641 /* This ensures there is enough space in the command stream. */
642 void (*need_gfx_cs_space)(struct pipe_context *ctx, unsigned num_dw,
643 bool include_draw_vbo);
644
645 void (*set_atom_dirty)(struct r600_common_context *ctx,
646 struct r600_atom *atom, bool dirty);
647
648 void (*check_vm_faults)(struct r600_common_context *ctx,
649 struct radeon_saved_cs *saved,
650 enum ring_type ring);
651 };
652
653 /* r600_buffer.c */
654 bool r600_rings_is_buffer_referenced(struct r600_common_context *ctx,
655 struct pb_buffer *buf,
656 enum radeon_bo_usage usage);
657 void *r600_buffer_map_sync_with_rings(struct r600_common_context *ctx,
658 struct r600_resource *resource,
659 unsigned usage);
660 void r600_buffer_subdata(struct pipe_context *ctx,
661 struct pipe_resource *buffer,
662 unsigned usage, unsigned offset,
663 unsigned size, const void *data);
664 void r600_init_resource_fields(struct r600_common_screen *rscreen,
665 struct r600_resource *res,
666 uint64_t size, unsigned alignment);
667 bool r600_alloc_resource(struct r600_common_screen *rscreen,
668 struct r600_resource *res);
669 struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
670 const struct pipe_resource *templ,
671 unsigned alignment);
672 struct pipe_resource * r600_aligned_buffer_create(struct pipe_screen *screen,
673 unsigned bind,
674 unsigned usage,
675 unsigned size,
676 unsigned alignment);
677 struct pipe_resource *
678 r600_buffer_from_user_memory(struct pipe_screen *screen,
679 const struct pipe_resource *templ,
680 void *user_memory);
681 void
682 r600_invalidate_resource(struct pipe_context *ctx,
683 struct pipe_resource *resource);
684
685 /* r600_common_pipe.c */
686 void r600_draw_rectangle(struct blitter_context *blitter,
687 int x1, int y1, int x2, int y2, float depth,
688 enum blitter_attrib_type type,
689 const union pipe_color_union *attrib);
690 bool r600_common_screen_init(struct r600_common_screen *rscreen,
691 struct radeon_winsys *ws);
692 void r600_destroy_common_screen(struct r600_common_screen *rscreen);
693 void r600_preflush_suspend_features(struct r600_common_context *ctx);
694 void r600_postflush_resume_features(struct r600_common_context *ctx);
695 bool r600_common_context_init(struct r600_common_context *rctx,
696 struct r600_common_screen *rscreen,
697 unsigned context_flags);
698 void r600_common_context_cleanup(struct r600_common_context *rctx);
699 void r600_context_add_resource_size(struct pipe_context *ctx, struct pipe_resource *r);
700 bool r600_can_dump_shader(struct r600_common_screen *rscreen,
701 unsigned processor);
702 void r600_screen_clear_buffer(struct r600_common_screen *rscreen, struct pipe_resource *dst,
703 uint64_t offset, uint64_t size, unsigned value,
704 enum r600_coherency coher);
705 struct pipe_resource *r600_resource_create_common(struct pipe_screen *screen,
706 const struct pipe_resource *templ);
707 const char *r600_get_llvm_processor_name(enum radeon_family family);
708 void r600_need_dma_space(struct r600_common_context *ctx, unsigned num_dw,
709 struct r600_resource *dst, struct r600_resource *src);
710 void r600_dma_emit_wait_idle(struct r600_common_context *rctx);
711 void radeon_save_cs(struct radeon_winsys *ws, struct radeon_winsys_cs *cs,
712 struct radeon_saved_cs *saved);
713 void radeon_clear_saved_cs(struct radeon_saved_cs *saved);
714
715 /* r600_gpu_load.c */
716 void r600_gpu_load_kill_thread(struct r600_common_screen *rscreen);
717 uint64_t r600_gpu_load_begin(struct r600_common_screen *rscreen);
718 unsigned r600_gpu_load_end(struct r600_common_screen *rscreen, uint64_t begin);
719
720 /* r600_perfcounters.c */
721 void r600_perfcounters_destroy(struct r600_common_screen *rscreen);
722
723 /* r600_query.c */
724 void r600_init_screen_query_functions(struct r600_common_screen *rscreen);
725 void r600_query_init(struct r600_common_context *rctx);
726 void r600_suspend_queries(struct r600_common_context *ctx);
727 void r600_resume_queries(struct r600_common_context *ctx);
728 void r600_query_init_backend_mask(struct r600_common_context *ctx);
729
730 /* r600_streamout.c */
731 void r600_streamout_buffers_dirty(struct r600_common_context *rctx);
732 void r600_set_streamout_targets(struct pipe_context *ctx,
733 unsigned num_targets,
734 struct pipe_stream_output_target **targets,
735 const unsigned *offset);
736 void r600_emit_streamout_end(struct r600_common_context *rctx);
737 void r600_update_prims_generated_query_state(struct r600_common_context *rctx,
738 unsigned type, int diff);
739 void r600_streamout_init(struct r600_common_context *rctx);
740
741 /* r600_test_dma.c */
742 void r600_test_dma(struct r600_common_screen *rscreen);
743
744 /* r600_texture.c */
745 bool r600_prepare_for_dma_blit(struct r600_common_context *rctx,
746 struct r600_texture *rdst,
747 unsigned dst_level, unsigned dstx,
748 unsigned dsty, unsigned dstz,
749 struct r600_texture *rsrc,
750 unsigned src_level,
751 const struct pipe_box *src_box);
752 void r600_texture_get_fmask_info(struct r600_common_screen *rscreen,
753 struct r600_texture *rtex,
754 unsigned nr_samples,
755 struct r600_fmask_info *out);
756 void r600_texture_get_cmask_info(struct r600_common_screen *rscreen,
757 struct r600_texture *rtex,
758 struct r600_cmask_info *out);
759 bool r600_init_flushed_depth_texture(struct pipe_context *ctx,
760 struct pipe_resource *texture,
761 struct r600_texture **staging);
762 void r600_print_texture_info(struct r600_texture *rtex, FILE *f);
763 struct pipe_resource *r600_texture_create(struct pipe_screen *screen,
764 const struct pipe_resource *templ);
765 bool vi_dcc_formats_compatible(enum pipe_format format1,
766 enum pipe_format format2);
767 void vi_dcc_disable_if_incompatible_format(struct r600_common_context *rctx,
768 struct pipe_resource *tex,
769 unsigned level,
770 enum pipe_format view_format);
771 struct pipe_surface *r600_create_surface_custom(struct pipe_context *pipe,
772 struct pipe_resource *texture,
773 const struct pipe_surface *templ,
774 unsigned width, unsigned height);
775 unsigned r600_translate_colorswap(enum pipe_format format, bool do_endian_swap);
776 void vi_separate_dcc_start_query(struct pipe_context *ctx,
777 struct r600_texture *tex);
778 void vi_separate_dcc_stop_query(struct pipe_context *ctx,
779 struct r600_texture *tex);
780 void vi_separate_dcc_process_and_reset_stats(struct pipe_context *ctx,
781 struct r600_texture *tex);
782 void vi_dcc_clear_level(struct r600_common_context *rctx,
783 struct r600_texture *rtex,
784 unsigned level, unsigned clear_value);
785 void evergreen_do_fast_color_clear(struct r600_common_context *rctx,
786 struct pipe_framebuffer_state *fb,
787 struct r600_atom *fb_state,
788 unsigned *buffers, unsigned *dirty_cbufs,
789 const union pipe_color_union *color);
790 bool r600_texture_disable_dcc(struct r600_common_context *rctx,
791 struct r600_texture *rtex);
792 void r600_init_screen_texture_functions(struct r600_common_screen *rscreen);
793 void r600_init_context_texture_functions(struct r600_common_context *rctx);
794
795 /* r600_viewport.c */
796 void evergreen_apply_scissor_bug_workaround(struct r600_common_context *rctx,
797 struct pipe_scissor_state *scissor);
798 void r600_viewport_set_rast_deps(struct r600_common_context *rctx,
799 bool scissor_enable, bool clip_halfz);
800 void r600_update_vs_writes_viewport_index(struct r600_common_context *rctx,
801 struct tgsi_shader_info *info);
802 void r600_init_viewport_functions(struct r600_common_context *rctx);
803
804 /* cayman_msaa.c */
805 extern const uint32_t eg_sample_locs_2x[4];
806 extern const unsigned eg_max_dist_2x;
807 extern const uint32_t eg_sample_locs_4x[4];
808 extern const unsigned eg_max_dist_4x;
809 void cayman_get_sample_position(struct pipe_context *ctx, unsigned sample_count,
810 unsigned sample_index, float *out_value);
811 void cayman_init_msaa(struct pipe_context *ctx);
812 void cayman_emit_msaa_sample_locs(struct radeon_winsys_cs *cs, int nr_samples);
813 void cayman_emit_msaa_config(struct radeon_winsys_cs *cs, int nr_samples,
814 int ps_iter_samples, int overrast_samples,
815 unsigned sc_mode_cntl_1);
816
817
818 /* Inline helpers. */
819
820 static inline struct r600_resource *r600_resource(struct pipe_resource *r)
821 {
822 return (struct r600_resource*)r;
823 }
824
825 static inline void
826 r600_resource_reference(struct r600_resource **ptr, struct r600_resource *res)
827 {
828 pipe_resource_reference((struct pipe_resource **)ptr,
829 (struct pipe_resource *)res);
830 }
831
832 static inline void
833 r600_texture_reference(struct r600_texture **ptr, struct r600_texture *res)
834 {
835 pipe_resource_reference((struct pipe_resource **)ptr, &res->resource.b.b);
836 }
837
838 static inline bool r600_get_strmout_en(struct r600_common_context *rctx)
839 {
840 return rctx->streamout.streamout_enabled ||
841 rctx->streamout.prims_gen_query_enabled;
842 }
843
844 #define SQ_TEX_XY_FILTER_POINT 0x00
845 #define SQ_TEX_XY_FILTER_BILINEAR 0x01
846 #define SQ_TEX_XY_FILTER_ANISO_POINT 0x02
847 #define SQ_TEX_XY_FILTER_ANISO_BILINEAR 0x03
848
849 static inline unsigned eg_tex_filter(unsigned filter, unsigned max_aniso)
850 {
851 if (filter == PIPE_TEX_FILTER_LINEAR)
852 return max_aniso > 1 ? SQ_TEX_XY_FILTER_ANISO_BILINEAR
853 : SQ_TEX_XY_FILTER_BILINEAR;
854 else
855 return max_aniso > 1 ? SQ_TEX_XY_FILTER_ANISO_POINT
856 : SQ_TEX_XY_FILTER_POINT;
857 }
858
859 static inline unsigned r600_tex_aniso_filter(unsigned filter)
860 {
861 if (filter < 2)
862 return 0;
863 if (filter < 4)
864 return 1;
865 if (filter < 8)
866 return 2;
867 if (filter < 16)
868 return 3;
869 return 4;
870 }
871
872 static inline unsigned r600_wavefront_size(enum radeon_family family)
873 {
874 switch (family) {
875 case CHIP_RV610:
876 case CHIP_RS780:
877 case CHIP_RV620:
878 case CHIP_RS880:
879 return 16;
880 case CHIP_RV630:
881 case CHIP_RV635:
882 case CHIP_RV730:
883 case CHIP_RV710:
884 case CHIP_PALM:
885 case CHIP_CEDAR:
886 return 32;
887 default:
888 return 64;
889 }
890 }
891
892 static inline enum radeon_bo_priority
893 r600_get_sampler_view_priority(struct r600_resource *res)
894 {
895 if (res->b.b.target == PIPE_BUFFER)
896 return RADEON_PRIO_SAMPLER_BUFFER;
897
898 if (res->b.b.nr_samples > 1)
899 return RADEON_PRIO_SAMPLER_TEXTURE_MSAA;
900
901 return RADEON_PRIO_SAMPLER_TEXTURE;
902 }
903
904 static inline bool
905 r600_can_sample_zs(struct r600_texture *tex, bool stencil_sampler)
906 {
907 return (stencil_sampler && tex->can_sample_s) ||
908 (!stencil_sampler && tex->can_sample_z);
909 }
910
911 #define COMPUTE_DBG(rscreen, fmt, args...) \
912 do { \
913 if ((rscreen->b.debug_flags & DBG_COMPUTE)) fprintf(stderr, fmt, ##args); \
914 } while (0);
915
916 #define R600_ERR(fmt, args...) \
917 fprintf(stderr, "EE %s:%d %s - " fmt, __FILE__, __LINE__, __func__, ##args)
918
919 /* For MSAA sample positions. */
920 #define FILL_SREG(s0x, s0y, s1x, s1y, s2x, s2y, s3x, s3y) \
921 (((s0x) & 0xf) | (((unsigned)(s0y) & 0xf) << 4) | \
922 (((unsigned)(s1x) & 0xf) << 8) | (((unsigned)(s1y) & 0xf) << 12) | \
923 (((unsigned)(s2x) & 0xf) << 16) | (((unsigned)(s2y) & 0xf) << 20) | \
924 (((unsigned)(s3x) & 0xf) << 24) | (((unsigned)(s3y) & 0xf) << 28))
925
926 #endif