#define SI_CONTEXT_FLUSH_FOR_RENDER_COND (1 << 2)
/* Instruction cache. */
#define SI_CONTEXT_INV_ICACHE (1 << 3)
-/* SMEM L1, other names: KCACHE, constant cache, DCACHE, data cache */
-#define SI_CONTEXT_INV_SMEM_L1 (1 << 4)
-/* VMEM L1 can optionally be bypassed (GLC=1). Other names: TC L1 */
-#define SI_CONTEXT_INV_VMEM_L1 (1 << 5)
-/* Used by everything except CB/DB, can be bypassed (SLC=1). Other names: TC L2 */
-#define SI_CONTEXT_INV_GLOBAL_L2 (1 << 6)
-/* Write dirty L2 lines back to memory (shader and CP DMA stores), but don't
- * invalidate L2. GFX6-GFX7 can't do it, so they will do complete invalidation. */
-#define SI_CONTEXT_WRITEBACK_GLOBAL_L2 (1 << 7)
-/* Writeback & invalidate the L2 metadata cache. It can only be coupled with
+/* Scalar cache. (GFX6-9: scalar L1; GFX10: scalar L0)
+ * GFX10: This also invalidates the L1 shader array cache. */
+#define SI_CONTEXT_INV_SCACHE (1 << 4)
+/* Vector cache. (GFX6-9: vector L1; GFX10: vector L0)
+ * GFX10: This also invalidates the L1 shader array cache. */
+#define SI_CONTEXT_INV_VCACHE (1 << 5)
+/* L2 cache + L2 metadata cache writeback & invalidate.
+ * GFX6-8: Used by shaders only. GFX9-10: Used by everything. */
+#define SI_CONTEXT_INV_L2 (1 << 6)
+/* L2 writeback (write dirty L2 lines to memory for non-L2 clients).
+ * Only used for coherency with non-L2 clients like CB, DB, CP on GFX6-8.
+ * GFX6-7 will do complete invalidation, because the writeback is unsupported. */
+#define SI_CONTEXT_WB_L2 (1 << 7)
+/* Writeback & invalidate the L2 metadata cache only. It can only be coupled with
* a CB or DB flush. */
#define SI_CONTEXT_INV_L2_METADATA (1 << 8)
/* Framebuffer caches. */
/* Shader compiler options the shader cache should be aware of: */
DBG_FS_CORRECT_DERIVS_AFTER_KILL,
- DBG_UNSAFE_MATH,
DBG_SI_SCHED,
DBG_GISEL,
+ DBG_W32_GE,
+ DBG_W32_PS,
+ DBG_W32_CS,
+ DBG_W64_GE,
+ DBG_W64_PS,
+ DBG_W64_CS,
/* Shader compiler options (with no effect on the shader cache): */
DBG_CHECK_IR,
DBG_ZERO_VRAM,
/* 3D engine options: */
+ DBG_NO_GFX,
+ DBG_NO_NGG,
DBG_ALWAYS_PD,
DBG_PD,
DBG_NO_PD,
enum pipe_format db_render_format:16;
uint8_t stencil_clear_value;
bool tc_compatible_htile:1;
+ bool htile_stencil_disabled:1;
bool depth_cleared:1; /* if it was cleared at least once */
bool stencil_cleared:1; /* if it was cleared at least once */
bool upgraded_depth:1; /* upgraded from unorm to Z32_FLOAT */
unsigned cb_color_view;
unsigned cb_color_attrib;
unsigned cb_color_attrib2; /* GFX9 and later */
+ unsigned cb_color_attrib3; /* GFX10 and later */
unsigned cb_dcc_control; /* GFX8 and later */
unsigned spi_shader_col_format:8; /* no blending, no alpha-to-coverage. */
unsigned spi_shader_col_format_alpha:8; /* alpha-to-coverage */
uint64_t db_htile_data_base;
unsigned db_depth_info;
unsigned db_z_info;
- unsigned db_z_info2; /* GFX9+ */
+ unsigned db_z_info2; /* GFX9 only */
unsigned db_depth_view;
unsigned db_depth_size;
unsigned db_depth_slice;
unsigned db_stencil_info;
- unsigned db_stencil_info2; /* GFX9+ */
+ unsigned db_stencil_info2; /* GFX9 only */
unsigned db_htile_surface;
};
uint64_t debug_flags;
char renderer_string[183];
+ void (*make_texture_descriptor)(
+ struct si_screen *screen,
+ struct si_texture *tex,
+ bool sampler,
+ enum pipe_texture_target target,
+ enum pipe_format pipe_format,
+ const unsigned char state_swizzle[4],
+ unsigned first_level, unsigned last_level,
+ unsigned first_layer, unsigned last_layer,
+ unsigned width, unsigned height, unsigned depth,
+ uint32_t *state,
+ uint32_t *fmask_state);
+
unsigned pa_sc_raster_config;
unsigned pa_sc_raster_config_1;
unsigned se_tile_repeat;
unsigned eqaa_force_coverage_samples;
unsigned eqaa_force_z_samples;
unsigned eqaa_force_color_samples;
- bool has_clear_state;
- bool has_distributed_tess;
bool has_draw_indirect_multi;
bool has_out_of_order_rast;
bool assume_no_z_fights;
bool commutative_blend_add;
- bool has_gfx9_scissor_bug;
- bool has_msaa_sample_loc_bug;
- bool has_ls_vgpr_init_bug;
- bool has_dcc_constant_encode;
bool dpbb_allowed;
bool dfsm_allowed;
bool llvm_has_working_vgpr_indexing;
+ bool use_ngg;
+ bool use_ngg_streamout;
struct {
#define OPT_BOOL(name, dflt, description) bool name:1;
/* Whether shaders are monolithic (1-part) or separate (3-part). */
bool use_monolithic_shaders;
bool record_llvm_ir;
- bool has_rbplus; /* if RB+ registers exist */
- bool rbplus_allowed; /* if RB+ is allowed */
bool dcc_msaa_allowed;
- bool cpdma_prefetch_writes_memory;
struct slab_parent_pool pool_transfers;
/* Use at most 2 low priority threads on quadcore and better.
* We want to minimize the impact on multithreaded Mesa. */
struct ac_llvm_compiler compiler_lowp[10];
+
+ unsigned compute_wave_size;
+ unsigned ps_wave_size;
+ unsigned ge_wave_size;
};
struct si_blend_color {
#define SI_SAMPLER_STATE_MAGIC 0x34f1c35a
struct si_sampler_state {
-#ifdef DEBUG
+#ifndef NDEBUG
unsigned magic;
#endif
uint32_t val[4];
ubyte color_is_int10;
ubyte dirty_cbufs;
ubyte dcc_overwrite_combiner_watermark;
+ ubyte min_bytes_per_pixel;
bool dirty_zsbuf;
bool any_dst_linear;
bool CB_has_shader_readable_metadata;
uint32_t index;
};
+#define SI_NUM_VGT_STAGES_KEY_BITS 4
+#define SI_NUM_VGT_STAGES_STATES (1 << SI_NUM_VGT_STAGES_KEY_BITS)
+
+/* The VGT_SHADER_STAGES key used to index the table of precomputed values.
+ * Some fields are set by state-change calls, most are set by draw_vbo.
+ */
+union si_vgt_stages_key {
+ struct {
+#ifdef PIPE_ARCH_LITTLE_ENDIAN
+ unsigned tess:1;
+ unsigned gs:1;
+ unsigned ngg:1; /* gfx10+ */
+ unsigned streamout:1; /* only used with NGG */
+ unsigned _pad:32 - SI_NUM_VGT_STAGES_KEY_BITS;
+#else /* PIPE_ARCH_BIG_ENDIAN */
+ unsigned _pad:32 - SI_NUM_VGT_STAGES_KEY_BITS;
+ unsigned streamout:1;
+ unsigned ngg:1;
+ unsigned gs:1;
+ unsigned tess:1;
+#endif
+ } u;
+ uint32_t index;
+};
+
struct si_texture_handle
{
unsigned desc_slot;
struct pipe_device_reset_callback device_reset_callback;
struct u_log_context *log;
void *query_result_shader;
+ void *sh_query_result_shader;
+
+ void (*emit_cache_flush)(struct si_context *ctx);
+
struct blitter_context *blitter;
+ void *noop_blend;
+ void *noop_dsa;
+ void *discard_rasterizer_state;
void *custom_dsa_flush;
void *custom_blend_resolve;
void *custom_blend_fmask_decompress;
unsigned num_gfx_cs_flushes;
unsigned initial_gfx_cs_size;
- unsigned gpu_reset_counter;
unsigned last_dirty_tex_counter;
unsigned last_dirty_buf_counter;
unsigned last_compressed_colortex_counter;
struct si_pm4_state *init_config;
struct si_pm4_state *init_config_gs_rings;
bool init_config_has_vgt_flush;
- struct si_pm4_state *vgt_shader_config[4];
+ struct si_pm4_state *vgt_shader_config[SI_NUM_VGT_STAGES_STATES];
/* shaders */
struct si_shader_ctx_state ps_shader;
bool gs_tri_strip_adj_fix:1;
bool ls_vgpr_fix:1;
bool prim_discard_cs_instancing:1;
+ bool ngg:1;
int last_index_size;
int last_base_vertex;
int last_start_instance;
int last_prim;
int last_multi_vgt_param;
int last_rast_prim;
+ int last_flatshade_first;
+ int last_binning_enabled;
unsigned last_sc_line_stipple;
unsigned current_vs_state;
unsigned last_vs_state;
struct si_resource *scratch_buffer;
unsigned scratch_waves;
unsigned spi_tmpring_size;
+ unsigned max_seen_scratch_bytes_per_wave;
+ unsigned max_seen_compute_scratch_bytes_per_wave;
struct si_resource *compute_scratch_buffer;
unsigned num_sdma_uploads;
unsigned max_sdma_uploads;
+ /* Shader-based queries. */
+ struct list_head shader_query_buffers;
+ unsigned num_active_shader_queries;
+
/* Statistics gathering for the DCC enablement heuristic. It can't be
* in si_texture because si_texture can be shared by multiple
* contexts. This is for back buffers only. We shouldn't get too many
unsigned src_level,
const struct pipe_box *src_box);
void si_decompress_dcc(struct si_context *sctx, struct si_texture *tex);
-void si_blit_decompress_depth(struct pipe_context *ctx,
- struct si_texture *texture,
- struct si_texture *staging,
- unsigned first_level, unsigned last_level,
- unsigned first_layer, unsigned last_layer,
- unsigned first_sample, unsigned last_sample);
/* si_buffer.c */
bool si_rings_is_buffer_referenced(struct si_context *sctx,
/* si_clear.c */
enum pipe_format si_simplify_cb_format(enum pipe_format format);
-bool vi_alpha_is_on_msb(enum pipe_format format);
-void vi_dcc_clear_level(struct si_context *sctx,
+bool vi_alpha_is_on_msb(struct si_screen *sscreen, enum pipe_format format);
+bool vi_dcc_clear_level(struct si_context *sctx,
struct si_texture *tex,
unsigned level, unsigned clear_value);
void si_init_clear_functions(struct si_context *sctx);
void si_init_debug_functions(struct si_context *sctx);
void si_check_vm_faults(struct si_context *sctx,
struct radeon_saved_cs *saved, enum ring_type ring);
-bool si_replace_shader(unsigned num, struct ac_shader_binary *binary);
+bool si_replace_shader(unsigned num, struct si_shader_binary *binary);
/* si_dma.c */
void si_init_dma_functions(struct si_context *sctx);
/* si_gfx_cs.c */
void si_flush_gfx_cs(struct si_context *ctx, unsigned flags,
struct pipe_fence_handle **fence);
+void si_allocate_gds(struct si_context *ctx);
void si_begin_new_gfx_cs(struct si_context *ctx);
void si_need_gfx_cs_space(struct si_context *ctx);
void si_unref_sdma_uploads(struct si_context *sctx);
/* si_compute.c */
void si_emit_initial_compute_regs(struct si_context *sctx, struct radeon_cmdbuf *cs);
-unsigned si_get_compute_resource_limits(struct si_screen *sscreen,
- unsigned waves_per_threadgroup,
- unsigned max_waves_per_sh,
- unsigned threadgroups_per_cu);
void si_init_compute_functions(struct si_context *sctx);
/* si_compute_prim_discard.c */
void *si_clear_render_target_shader_1d_array(struct pipe_context *ctx);
void *si_create_dcc_retile_cs(struct pipe_context *ctx);
void *si_create_query_result_cs(struct si_context *sctx);
+void *gfx10_create_sh_query_result_cs(struct si_context *sctx);
+
+/* gfx10_query.c */
+void gfx10_init_query(struct si_context *sctx);
+void gfx10_destroy_query(struct si_context *sctx);
/* si_test_dma.c */
void si_test_dma(struct si_screen *sscreen);
void si_texture_discard_cmask(struct si_screen *sscreen,
struct si_texture *tex);
bool si_init_flushed_depth_texture(struct pipe_context *ctx,
- struct pipe_resource *texture,
- struct si_texture **staging);
+ struct pipe_resource *texture);
void si_print_texture_info(struct si_screen *sscreen,
struct si_texture *tex, struct u_log_context *log);
struct pipe_resource *si_texture_create(struct pipe_screen *screen,
const struct pipe_resource *templ);
-bool vi_dcc_formats_compatible(enum pipe_format format1,
+bool vi_dcc_formats_compatible(struct si_screen *sscreen,
+ enum pipe_format format1,
enum pipe_format format2);
bool vi_dcc_formats_are_incompatible(struct pipe_resource *tex,
unsigned level,
static inline struct si_shader* si_get_vs_state(struct si_context *sctx)
{
- if (sctx->gs_shader.cso)
+ if (sctx->gs_shader.cso &&
+ sctx->gs_shader.current &&
+ !sctx->gs_shader.current->key.as_ngg)
return sctx->gs_shader.cso->gs_copy_shader;
struct si_shader_ctx_state *vs = si_get_vs(sctx);
bool shaders_read_metadata, bool dcc_pipe_aligned)
{
sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_CB |
- SI_CONTEXT_INV_VMEM_L1;
+ SI_CONTEXT_INV_VCACHE;
- if (sctx->chip_class >= GFX9) {
+ if (sctx->chip_class >= GFX10) {
+ if (shaders_read_metadata)
+ sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
+ } else if (sctx->chip_class == GFX9) {
/* Single-sample color is coherent with shaders on GFX9, but
* L2 metadata must be flushed if shaders read metadata.
* (DCC, CMASK).
*/
if (num_samples >= 2 ||
(shaders_read_metadata && !dcc_pipe_aligned))
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ sctx->flags |= SI_CONTEXT_INV_L2;
else if (shaders_read_metadata)
sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
} else {
/* GFX6-GFX8 */
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ sctx->flags |= SI_CONTEXT_INV_L2;
}
}
bool include_stencil, bool shaders_read_metadata)
{
sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_DB |
- SI_CONTEXT_INV_VMEM_L1;
+ SI_CONTEXT_INV_VCACHE;
- if (sctx->chip_class >= GFX9) {
+ if (sctx->chip_class >= GFX10) {
+ if (shaders_read_metadata)
+ sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
+ } else if (sctx->chip_class == GFX9) {
/* Single-sample depth (not stencil) is coherent with shaders
* on GFX9, but L2 metadata must be flushed if shaders read
* metadata.
*/
if (num_samples >= 2 || include_stencil)
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ sctx->flags |= SI_CONTEXT_INV_L2;
else if (shaders_read_metadata)
sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
} else {
/* GFX6-GFX8 */
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ sctx->flags |= SI_CONTEXT_INV_L2;
}
}
}
static inline bool
-si_htile_enabled(struct si_texture *tex, unsigned level)
+si_htile_enabled(struct si_texture *tex, unsigned level, unsigned zs_mask)
{
+ if (zs_mask == PIPE_MASK_S && tex->htile_stencil_disabled)
+ return false;
+
return tex->htile_offset && level == 0;
}
static inline bool
-vi_tc_compat_htile_enabled(struct si_texture *tex, unsigned level)
+vi_tc_compat_htile_enabled(struct si_texture *tex, unsigned level, unsigned zs_mask)
{
assert(!tex->tc_compatible_htile || tex->htile_offset);
- return tex->tc_compatible_htile && level == 0;
+ return tex->tc_compatible_htile && si_htile_enabled(tex, level, zs_mask);
}
static inline unsigned si_get_ps_iter_samples(struct si_context *sctx)
(1 << PIPE_PRIM_POINTS))) != 0;
}
+static inline bool util_rast_prim_is_triangles(unsigned prim)
+{
+ return ((1 << prim) & ((1 << PIPE_PRIM_TRIANGLES) |
+ (1 << PIPE_PRIM_TRIANGLE_STRIP) |
+ (1 << PIPE_PRIM_TRIANGLE_FAN) |
+ (1 << PIPE_PRIM_QUADS) |
+ (1 << PIPE_PRIM_QUAD_STRIP) |
+ (1 << PIPE_PRIM_POLYGON) |
+ (1 << PIPE_PRIM_TRIANGLES_ADJACENCY) |
+ (1 << PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY)));
+}
+
/**
* Return true if there is enough memory in VRAM and GTT for the buffers
* added so far.
return sctx->prim_discard_vertex_count_threshold != UINT_MAX;
}
+static inline unsigned si_get_wave_size(struct si_screen *sscreen,
+ enum pipe_shader_type shader_type,
+ bool ngg, bool es)
+{
+ if (shader_type == PIPE_SHADER_COMPUTE)
+ return sscreen->compute_wave_size;
+ else if (shader_type == PIPE_SHADER_FRAGMENT)
+ return sscreen->ps_wave_size;
+ else if ((shader_type == PIPE_SHADER_VERTEX && es && !ngg) ||
+ (shader_type == PIPE_SHADER_TESS_EVAL && es && !ngg) ||
+ (shader_type == PIPE_SHADER_GEOMETRY && !ngg)) /* legacy GS only supports Wave64 */
+ return 64;
+ else
+ return sscreen->ge_wave_size;
+}
+
+static inline unsigned si_get_shader_wave_size(struct si_shader *shader)
+{
+ return si_get_wave_size(shader->selector->screen, shader->selector->type,
+ shader->key.as_ngg, shader->key.as_es);
+}
+
#define PRINT_ERR(fmt, args...) \
fprintf(stderr, "EE %s:%d %s - " fmt, __FILE__, __LINE__, __func__, ##args)