#endif
#define ATI_VENDOR_ID 0x1002
-
+#define SI_PRIM_DISCARD_DEBUG 0
#define SI_NOT_QUERY 0xffffffff
/* The base vertex and primitive restart can be any number, but we must pick
#define SI_CONTEXT_FLUSH_FOR_RENDER_COND (1 << 2)
/* Instruction cache. */
#define SI_CONTEXT_INV_ICACHE (1 << 3)
-/* SMEM L1, other names: KCACHE, constant cache, DCACHE, data cache */
-#define SI_CONTEXT_INV_SMEM_L1 (1 << 4)
-/* VMEM L1 can optionally be bypassed (GLC=1). Other names: TC L1 */
-#define SI_CONTEXT_INV_VMEM_L1 (1 << 5)
-/* Used by everything except CB/DB, can be bypassed (SLC=1). Other names: TC L2 */
-#define SI_CONTEXT_INV_GLOBAL_L2 (1 << 6)
-/* Write dirty L2 lines back to memory (shader and CP DMA stores), but don't
- * invalidate L2. SI-CIK can't do it, so they will do complete invalidation. */
-#define SI_CONTEXT_WRITEBACK_GLOBAL_L2 (1 << 7)
-/* Writeback & invalidate the L2 metadata cache. It can only be coupled with
+/* Scalar L1 cache. */
+#define SI_CONTEXT_INV_SCACHE (1 << 4)
+/* Vector L1 cache. */
+#define SI_CONTEXT_INV_VCACHE (1 << 5)
+/* L2 cache + L2 metadata cache writeback & invalidate.
+ * GFX6-8: Used by shaders only. GFX9-10: Used by everything. */
+#define SI_CONTEXT_INV_L2 (1 << 6)
+/* L2 writeback (write dirty L2 lines to memory for non-L2 clients).
+ * Only used for coherency with non-L2 clients like CB, DB, CP on GFX6-8.
+ * GFX6-7 will do complete invalidation, because the writeback is unsupported. */
+#define SI_CONTEXT_WB_L2 (1 << 7)
+/* Writeback & invalidate the L2 metadata cache only. It can only be coupled with
* a CB or DB flush. */
#define SI_CONTEXT_INV_L2_METADATA (1 << 8)
/* Framebuffer caches. */
/* Shader compiler options (with no effect on the shader cache): */
DBG_CHECK_IR,
- DBG_NIR,
DBG_MONOLITHIC_SHADERS,
DBG_NO_OPT_VARIANT,
DBG_ZERO_VRAM,
/* 3D engine options: */
+ DBG_ALWAYS_PD,
+ DBG_PD,
+ DBG_NO_PD,
DBG_SWITCH_ON_EOP,
DBG_NO_OUT_OF_ORDER,
DBG_NO_DPBB,
};
struct si_compute;
+struct si_shader_context;
struct hash_table;
struct u_suballocator;
unsigned cb_color_view;
unsigned cb_color_attrib;
unsigned cb_color_attrib2; /* GFX9 and later */
- unsigned cb_dcc_control; /* VI and later */
+ unsigned cb_color_attrib3; /* GFX10 and later */
+ unsigned cb_dcc_control; /* GFX8 and later */
unsigned spi_shader_col_format:8; /* no blending, no alpha-to-coverage. */
unsigned spi_shader_col_format_alpha:8; /* alpha-to-coverage */
unsigned spi_shader_col_format_blend:8; /* blending without alpha. */
uint64_t db_htile_data_base;
unsigned db_depth_info;
unsigned db_z_info;
- unsigned db_z_info2; /* GFX9+ */
+ unsigned db_z_info2; /* GFX9 only */
unsigned db_depth_view;
unsigned db_depth_size;
unsigned db_depth_slice;
unsigned db_stencil_info;
- unsigned db_stencil_info2; /* GFX9+ */
+ unsigned db_stencil_info2; /* GFX9 only */
unsigned db_htile_surface;
};
uint64_t debug_flags;
char renderer_string[183];
+ void (*make_texture_descriptor)(
+ struct si_screen *screen,
+ struct si_texture *tex,
+ bool sampler,
+ enum pipe_texture_target target,
+ enum pipe_format pipe_format,
+ const unsigned char state_swizzle[4],
+ unsigned first_level, unsigned last_level,
+ unsigned first_layer, unsigned last_layer,
+ unsigned width, unsigned height, unsigned depth,
+ uint32_t *state,
+ uint32_t *fmask_state);
+
unsigned pa_sc_raster_config;
unsigned pa_sc_raster_config_1;
unsigned se_tile_repeat;
bool has_out_of_order_rast;
bool assume_no_z_fights;
bool commutative_blend_add;
- bool clear_db_cache_before_clear;
+ bool has_gfx9_scissor_bug;
bool has_msaa_sample_loc_bug;
bool has_ls_vgpr_init_bug;
bool has_dcc_constant_encode;
bool dfsm_allowed;
bool llvm_has_working_vgpr_indexing;
+ struct {
+#define OPT_BOOL(name, dflt, description) bool name:1;
+#include "si_debug_options.h"
+ } options;
+
/* Whether shaders are monolithic (1-part) or separate (3-part). */
bool use_monolithic_shaders;
bool record_llvm_ir;
* the counter before drawing and re-emit the states accordingly.
*/
unsigned dirty_tex_counter;
+ unsigned dirty_buf_counter;
/* Atomically increment this counter when an existing texture's
* metadata is enabled or disabled in a way that requires changing
#define SI_SAMPLER_STATE_MAGIC 0x34f1c35a
struct si_sampler_state {
-#ifdef DEBUG
+#ifndef NDEBUG
unsigned magic;
#endif
uint32_t val[4];
enum si_quant_mode quant_mode;
};
-struct si_scissors {
- unsigned dirty_mask;
- struct pipe_scissor_state states[SI_MAX_VIEWPORTS];
-};
-
struct si_viewports {
- unsigned dirty_mask;
- unsigned depth_range_dirty_mask;
struct pipe_viewport_state states[SI_MAX_VIEWPORTS];
struct si_signed_scissor as_scissor[SI_MAX_VIEWPORTS];
+ bool y_inverted;
};
struct si_clip_state {
uint32_t index;
};
+#define SI_NUM_VGT_STAGES_KEY_BITS 2
+#define SI_NUM_VGT_STAGES_STATES (1 << SI_NUM_VGT_STAGES_KEY_BITS)
+
+/* The VGT_SHADER_STAGES key used to index the table of precomputed values.
+ * Some fields are set by state-change calls, most are set by draw_vbo.
+ */
+union si_vgt_stages_key {
+ struct {
+#ifdef PIPE_ARCH_LITTLE_ENDIAN
+ unsigned tess:1;
+ unsigned gs:1;
+ unsigned _pad:32 - SI_NUM_VGT_STAGES_KEY_BITS;
+#else /* PIPE_ARCH_BIG_ENDIAN */
+ unsigned _pad:32 - SI_NUM_VGT_STAGES_KEY_BITS;
+ unsigned gs:1;
+ unsigned tess:1;
+#endif
+ } u;
+ uint32_t index;
+};
+
struct si_texture_handle
{
unsigned desc_slot;
struct pipe_reference reference;
struct si_context *ctx;
struct radeon_saved_cs gfx;
+ struct radeon_saved_cs compute;
struct si_resource *trace_buf;
unsigned trace_id;
unsigned gfx_last_dw;
+ unsigned compute_last_dw;
bool flushed;
int64_t time_flush;
};
struct pipe_device_reset_callback device_reset_callback;
struct u_log_context *log;
void *query_result_shader;
+
+ void (*emit_cache_flush)(struct si_context *ctx);
+
struct blitter_context *blitter;
void *custom_dsa_flush;
void *custom_blend_resolve;
struct pipe_debug_callback debug;
struct ac_llvm_compiler compiler; /* only non-threaded compilation */
struct si_shader_ctx_state fixed_func_tcs_shader;
+ /* Offset 0: EOP flush number; Offset 4: GDS prim restart counter */
struct si_resource *wait_mem_scratch;
unsigned wait_mem_number;
uint16_t prefetch_L2_mask;
unsigned num_gfx_cs_flushes;
unsigned initial_gfx_cs_size;
- unsigned gpu_reset_counter;
unsigned last_dirty_tex_counter;
+ unsigned last_dirty_buf_counter;
unsigned last_compressed_colortex_counter;
unsigned last_num_draw_calls;
unsigned flags; /* flush flags */
uint64_t vram;
uint64_t gtt;
+ /* Compute-based primitive discard. */
+ unsigned prim_discard_vertex_count_threshold;
+ struct pb_buffer *gds;
+ struct pb_buffer *gds_oa;
+ struct radeon_cmdbuf *prim_discard_compute_cs;
+ unsigned compute_gds_offset;
+ struct si_shader *compute_ib_last_shader;
+ uint32_t compute_rewind_va;
+ unsigned compute_num_prims_in_batch;
+ bool preserve_prim_restart_gds_at_flush;
+ /* index_ring is divided into 2 halves for doublebuffering. */
+ struct si_resource *index_ring;
+ unsigned index_ring_base; /* offset of a per-IB portion */
+ unsigned index_ring_offset; /* offset within a per-IB portion */
+ unsigned index_ring_size_per_ib; /* max available size per IB */
+ bool prim_discard_compute_ib_initialized;
+ /* For tracking the last execution barrier - it can be either
+ * a WRITE_DATA packet or a fence. */
+ uint32_t *last_pkt3_write_data;
+ struct si_resource *barrier_buf;
+ unsigned barrier_buf_offset;
+ struct pipe_fence_handle *last_ib_barrier_fence;
+ struct si_resource *last_ib_barrier_buf;
+ unsigned last_ib_barrier_buf_offset;
+
/* Atoms (direct states). */
union si_state_atoms atoms;
unsigned dirty_atoms; /* mask */
struct si_clip_state clip_state;
struct si_shader_data shader_pointers;
struct si_stencil_ref stencil_ref;
- struct si_scissors scissors;
+ struct pipe_scissor_state scissors[SI_MAX_VIEWPORTS];
struct si_streamout streamout;
struct si_viewports viewports;
unsigned num_window_rectangles;
struct si_pm4_state *init_config;
struct si_pm4_state *init_config_gs_rings;
bool init_config_has_vgt_flush;
- struct si_pm4_state *vgt_shader_config[4];
+ struct si_pm4_state *vgt_shader_config[SI_NUM_VGT_STAGES_STATES];
/* shaders */
struct si_shader_ctx_state ps_shader;
struct si_shader_ctx_state vs_shader;
struct si_shader_ctx_state tcs_shader;
struct si_shader_ctx_state tes_shader;
+ struct si_shader_ctx_state cs_prim_discard_state;
struct si_cs_shader_state cs_shader_state;
/* shader information */
bool bo_list_add_all_compute_resources;
/* other shader resources */
- struct pipe_constant_buffer null_const_buf; /* used for set_constant_buffer(NULL) on CIK */
+ struct pipe_constant_buffer null_const_buf; /* used for set_constant_buffer(NULL) on GFX7 */
struct pipe_resource *esgs_ring;
struct pipe_resource *gsvs_ring;
struct pipe_resource *tess_rings;
bool vertex_buffers_dirty;
bool vertex_buffer_pointer_dirty;
struct pipe_vertex_buffer vertex_buffer[SI_NUM_VERTEX_BUFFERS];
+ uint16_t vertex_buffer_unaligned; /* bitmask of not dword-aligned buffers */
/* MSAA config state. */
int ps_iter_samples;
/* Emitted draw state. */
bool gs_tri_strip_adj_fix:1;
bool ls_vgpr_fix:1;
+ bool prim_discard_cs_instancing:1;
int last_index_size;
int last_base_vertex;
int last_start_instance;
unsigned num_resident_handles;
uint64_t num_alloc_tex_transfer_bytes;
unsigned last_tex_ps_draw_ratio; /* for query */
- unsigned context_roll_counter;
+ unsigned compute_num_verts_accepted;
+ unsigned compute_num_verts_rejected;
+ unsigned compute_num_verts_ineligible; /* due to low vertex count */
+ unsigned context_roll;
/* Queries. */
/* Maintain the list of active queries for pausing between IBs. */
int num_occlusion_queries;
int num_perfect_occlusion_queries;
+ int num_pipeline_stat_queries;
struct list_head active_queries;
unsigned num_cs_dw_queries_suspend;
unsigned src_level,
const struct pipe_box *src_box);
void si_decompress_dcc(struct si_context *sctx, struct si_texture *tex);
-void si_blit_decompress_depth(struct pipe_context *ctx,
- struct si_texture *texture,
- struct si_texture *staging,
- unsigned first_level, unsigned last_level,
- unsigned first_layer, unsigned last_layer,
- unsigned first_sample, unsigned last_sample);
/* si_buffer.c */
bool si_rings_is_buffer_referenced(struct si_context *sctx,
void si_cp_write_data(struct si_context *sctx, struct si_resource *buf,
unsigned offset, unsigned size, unsigned dst_sel,
unsigned engine, const void *data);
-void si_cp_copy_data(struct si_context *sctx,
+void si_cp_copy_data(struct si_context *sctx, struct radeon_cmdbuf *cs,
unsigned dst_sel, struct si_resource *dst, unsigned dst_offset,
unsigned src_sel, struct si_resource *src, unsigned src_offset);
void si_init_debug_functions(struct si_context *sctx);
void si_check_vm_faults(struct si_context *sctx,
struct radeon_saved_cs *saved, enum ring_type ring);
-bool si_replace_shader(unsigned num, struct ac_shader_binary *binary);
+bool si_replace_shader(unsigned num, struct si_shader_binary *binary);
/* si_dma.c */
void si_init_dma_functions(struct si_context *sctx);
uint64_t offset, uint64_t size, unsigned value);
/* si_fence.c */
-void si_cp_release_mem(struct si_context *ctx,
+void si_cp_release_mem(struct si_context *ctx, struct radeon_cmdbuf *cs,
unsigned event, unsigned event_flags,
unsigned dst_sel, unsigned int_sel, unsigned data_sel,
struct si_resource *buf, uint64_t va,
uint64_t begin);
/* si_compute.c */
+void si_emit_initial_compute_regs(struct si_context *sctx, struct radeon_cmdbuf *cs);
+unsigned si_get_compute_resource_limits(struct si_screen *sscreen,
+ unsigned waves_per_threadgroup,
+ unsigned max_waves_per_sh,
+ unsigned threadgroups_per_cu);
void si_init_compute_functions(struct si_context *sctx);
+/* si_compute_prim_discard.c */
+enum si_prim_discard_outcome {
+ SI_PRIM_DISCARD_ENABLED,
+ SI_PRIM_DISCARD_DISABLED,
+ SI_PRIM_DISCARD_DRAW_SPLIT,
+};
+
+void si_build_prim_discard_compute_shader(struct si_shader_context *ctx);
+enum si_prim_discard_outcome
+si_prepare_prim_discard_or_split_draw(struct si_context *sctx,
+ const struct pipe_draw_info *info,
+ bool primitive_restart);
+void si_compute_signal_gfx(struct si_context *sctx);
+void si_dispatch_prim_discard_cs_and_draw(struct si_context *sctx,
+ const struct pipe_draw_info *info,
+ unsigned index_size,
+ unsigned base_vertex,
+ uint64_t input_indexbuf_va,
+ unsigned input_indexbuf_max_elements);
+void si_initialize_prim_discard_tunables(struct si_context *sctx);
+
/* si_perfcounters.c */
void si_init_perfcounters(struct si_screen *screen);
void si_destroy_perfcounters(struct si_screen *screen);
void si_texture_discard_cmask(struct si_screen *sscreen,
struct si_texture *tex);
bool si_init_flushed_depth_texture(struct pipe_context *ctx,
- struct pipe_resource *texture,
- struct si_texture **staging);
+ struct pipe_resource *texture);
void si_print_texture_info(struct si_screen *sscreen,
struct si_texture *tex, struct u_log_context *log);
struct pipe_resource *si_texture_create(struct pipe_screen *screen,
bool shaders_read_metadata, bool dcc_pipe_aligned)
{
sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_CB |
- SI_CONTEXT_INV_VMEM_L1;
+ SI_CONTEXT_INV_VCACHE;
if (sctx->chip_class >= GFX9) {
/* Single-sample color is coherent with shaders on GFX9, but
*/
if (num_samples >= 2 ||
(shaders_read_metadata && !dcc_pipe_aligned))
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ sctx->flags |= SI_CONTEXT_INV_L2;
else if (shaders_read_metadata)
sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
} else {
- /* SI-CI-VI */
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ /* GFX6-GFX8 */
+ sctx->flags |= SI_CONTEXT_INV_L2;
}
}
bool include_stencil, bool shaders_read_metadata)
{
sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_DB |
- SI_CONTEXT_INV_VMEM_L1;
+ SI_CONTEXT_INV_VCACHE;
if (sctx->chip_class >= GFX9) {
/* Single-sample depth (not stencil) is coherent with shaders
* metadata.
*/
if (num_samples >= 2 || include_stencil)
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ sctx->flags |= SI_CONTEXT_INV_L2;
else if (shaders_read_metadata)
sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
} else {
- /* SI-CI-VI */
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ /* GFX6-GFX8 */
+ sctx->flags |= SI_CONTEXT_INV_L2;
}
}
radeon_add_to_buffer_list(sctx, sctx->gfx_cs, bo, usage, priority);
}
+static inline bool si_compute_prim_discard_enabled(struct si_context *sctx)
+{
+ return sctx->prim_discard_vertex_count_threshold != UINT_MAX;
+}
+
#define PRINT_ERR(fmt, args...) \
fprintf(stderr, "EE %s:%d %s - " fmt, __FILE__, __LINE__, __func__, ##args)
projects / mesa.git / blobdiff