#include "util/u_idalloc.h"
#include "util/u_threaded_context.h"
-#ifdef PIPE_ARCH_BIG_ENDIAN
+#if UTIL_ARCH_BIG_ENDIAN
#define SI_BIG_ENDIAN 1
#else
#define SI_BIG_ENDIAN 0
#endif
#define ATI_VENDOR_ID 0x1002
-
+#define SI_PRIM_DISCARD_DEBUG 0
#define SI_NOT_QUERY 0xffffffff
/* The base vertex and primitive restart can be any number, but we must pick
#define SI_CONTEXT_FLUSH_FOR_RENDER_COND (1 << 2)
/* Instruction cache. */
#define SI_CONTEXT_INV_ICACHE (1 << 3)
-/* SMEM L1, other names: KCACHE, constant cache, DCACHE, data cache */
-#define SI_CONTEXT_INV_SMEM_L1 (1 << 4)
-/* VMEM L1 can optionally be bypassed (GLC=1). Other names: TC L1 */
-#define SI_CONTEXT_INV_VMEM_L1 (1 << 5)
-/* Used by everything except CB/DB, can be bypassed (SLC=1). Other names: TC L2 */
-#define SI_CONTEXT_INV_GLOBAL_L2 (1 << 6)
-/* Write dirty L2 lines back to memory (shader and CP DMA stores), but don't
- * invalidate L2. SI-CIK can't do it, so they will do complete invalidation. */
-#define SI_CONTEXT_WRITEBACK_GLOBAL_L2 (1 << 7)
-/* Writeback & invalidate the L2 metadata cache. It can only be coupled with
+/* Scalar cache. (GFX6-9: scalar L1; GFX10: scalar L0)
+ * GFX10: This also invalidates the L1 shader array cache. */
+#define SI_CONTEXT_INV_SCACHE (1 << 4)
+/* Vector cache. (GFX6-9: vector L1; GFX10: vector L0)
+ * GFX10: This also invalidates the L1 shader array cache. */
+#define SI_CONTEXT_INV_VCACHE (1 << 5)
+/* L2 cache + L2 metadata cache writeback & invalidate.
+ * GFX6-8: Used by shaders only. GFX9-10: Used by everything. */
+#define SI_CONTEXT_INV_L2 (1 << 6)
+/* L2 writeback (write dirty L2 lines to memory for non-L2 clients).
+ * Only used for coherency with non-L2 clients like CB, DB, CP on GFX6-8.
+ * GFX6-7 will do complete invalidation, because the writeback is unsupported. */
+#define SI_CONTEXT_WB_L2 (1 << 7)
+/* Writeback & invalidate the L2 metadata cache only. It can only be coupled with
* a CB or DB flush. */
#define SI_CONTEXT_INV_L2_METADATA (1 << 8)
/* Framebuffer caches. */
#define SI_RESOURCE_FLAG_UNMAPPABLE (PIPE_RESOURCE_FLAG_DRV_PRIV << 4)
#define SI_RESOURCE_FLAG_READ_ONLY (PIPE_RESOURCE_FLAG_DRV_PRIV << 5)
#define SI_RESOURCE_FLAG_32BIT (PIPE_RESOURCE_FLAG_DRV_PRIV << 6)
-#define SI_RESOURCE_FLAG_SO_FILLED_SIZE (PIPE_RESOURCE_FLAG_DRV_PRIV << 7)
+#define SI_RESOURCE_FLAG_CLEAR (PIPE_RESOURCE_FLAG_DRV_PRIV << 7)
+/* For const_uploader, upload data via GTT and copy to VRAM on context flush via SDMA. */
+#define SI_RESOURCE_FLAG_UPLOAD_FLUSH_EXPLICIT_VIA_SDMA (PIPE_RESOURCE_FLAG_DRV_PRIV << 8)
+
+enum si_clear_code
+{
+ DCC_CLEAR_COLOR_0000 = 0x00000000,
+ DCC_CLEAR_COLOR_0001 = 0x40404040,
+ DCC_CLEAR_COLOR_1110 = 0x80808080,
+ DCC_CLEAR_COLOR_1111 = 0xC0C0C0C0,
+ DCC_CLEAR_COLOR_REG = 0x20202020,
+ DCC_UNCOMPRESSED = 0xFFFFFFFF,
+};
+
+#define SI_IMAGE_ACCESS_AS_BUFFER (1 << 7)
/* Debug flags. */
enum {
/* Shader compiler options the shader cache should be aware of: */
DBG_FS_CORRECT_DERIVS_AFTER_KILL,
- DBG_UNSAFE_MATH,
DBG_SI_SCHED,
DBG_GISEL,
+ DBG_W32_GE,
+ DBG_W32_PS,
+ DBG_W32_CS,
+ DBG_W64_GE,
+ DBG_W64_PS,
+ DBG_W64_CS,
/* Shader compiler options (with no effect on the shader cache): */
DBG_CHECK_IR,
- DBG_NIR,
DBG_MONOLITHIC_SHADERS,
DBG_NO_OPT_VARIANT,
DBG_VM,
/* Driver options: */
- DBG_FORCE_DMA,
- DBG_NO_ASYNC_DMA,
+ DBG_FORCE_SDMA,
+ DBG_NO_SDMA,
+ DBG_NO_SDMA_CLEARS,
+ DBG_NO_SDMA_COPY_IMAGE,
DBG_NO_WC,
DBG_CHECK_VM,
DBG_RESERVE_VMID,
DBG_ZERO_VRAM,
/* 3D engine options: */
+ DBG_NO_GFX,
+ DBG_NO_NGG,
+ DBG_ALWAYS_PD,
+ DBG_PD,
+ DBG_NO_PD,
DBG_SWITCH_ON_EOP,
DBG_NO_OUT_OF_ORDER,
DBG_NO_DPBB,
};
struct si_compute;
+struct si_shader_context;
struct hash_table;
struct u_suballocator;
/* Only 32-bit buffer allocations are supported, gallium doesn't support more
* at the moment.
*/
-struct r600_resource {
+struct si_resource {
struct threaded_resource b;
/* Winsys objects. */
struct si_transfer {
struct threaded_transfer b;
- struct r600_resource *staging;
+ struct si_resource *staging;
unsigned offset;
};
struct si_texture {
- struct r600_resource buffer;
+ struct si_resource buffer;
struct radeon_surf surface;
- uint64_t size;
struct si_texture *flushed_depth_texture;
- /* Colorbuffer compression and fast clear. */
- uint64_t fmask_offset;
- uint64_t cmask_offset;
+ /* One texture allocation can contain these buffers:
+ * - image (pixel data)
+ * - FMASK buffer (MSAA compression)
+ * - CMASK buffer (MSAA compression and/or legacy fast color clear)
+ * - HTILE buffer (Z/S compression and fast Z/S clear)
+ * - DCC buffer (color compression and new fast color clear)
+ * - displayable DCC buffer (if the DCC buffer is not displayable)
+ * - DCC retile mapping buffer (if the DCC buffer is not displayable)
+ */
uint64_t cmask_base_address_reg;
- struct r600_resource *cmask_buffer;
- uint64_t dcc_offset; /* 0 = disabled */
+ struct si_resource *cmask_buffer;
unsigned cb_color_info; /* fast clear enable bit */
unsigned color_clear_value[2];
unsigned last_msaa_resolve_target_micro_mode;
unsigned num_level0_transfers;
+ unsigned plane_index; /* other planes are different pipe_resources */
+ unsigned num_planes;
/* Depth buffer compression and fast clear. */
- uint64_t htile_offset;
float depth_clear_value;
uint16_t dirty_level_mask; /* each bit says if that mipmap is compressed */
uint16_t stencil_dirty_level_mask; /* each bit says if that mipmap is compressed */
enum pipe_format db_render_format:16;
uint8_t stencil_clear_value;
+ bool fmask_is_not_identity:1;
bool tc_compatible_htile:1;
+ bool htile_stencil_disabled:1;
bool depth_cleared:1; /* if it was cleared at least once */
bool stencil_cleared:1; /* if it was cleared at least once */
bool upgraded_depth:1; /* upgraded from unorm to Z32_FLOAT */
* for a possible future enablement.
*/
bool separate_dcc_dirty:1;
+ bool displayable_dcc_dirty:1;
+
/* Statistics gathering for the DCC enablement heuristic. */
bool dcc_gather_statistics:1;
/* Counter that should be non-zero if the texture is bound to a
* target == 2D and last_level == 0. If enabled, dcc_offset contains
* the absolute GPUVM address, not the relative one.
*/
- struct r600_resource *dcc_separate_buffer;
+ struct si_resource *dcc_separate_buffer;
/* When DCC is temporarily disabled, the separate buffer is here. */
- struct r600_resource *last_dcc_separate_buffer;
+ struct si_resource *last_dcc_separate_buffer;
/* Estimate of how much this color buffer is written to in units of
* full-screen draws: ps_invocations / (width * height)
* Shader kills, late Z, and blending with trivial discards make it
unsigned cb_color_view;
unsigned cb_color_attrib;
unsigned cb_color_attrib2; /* GFX9 and later */
- unsigned cb_dcc_control; /* VI and later */
+ unsigned cb_color_attrib3; /* GFX10 and later */
+ unsigned cb_dcc_control; /* GFX8 and later */
unsigned spi_shader_col_format:8; /* no blending, no alpha-to-coverage. */
unsigned spi_shader_col_format_alpha:8; /* alpha-to-coverage */
unsigned spi_shader_col_format_blend:8; /* blending without alpha. */
uint64_t db_htile_data_base;
unsigned db_depth_info;
unsigned db_z_info;
- unsigned db_z_info2; /* GFX9+ */
+ unsigned db_z_info2; /* GFX9 only */
unsigned db_depth_view;
unsigned db_depth_size;
unsigned db_depth_slice;
unsigned db_stencil_info;
- unsigned db_stencil_info2; /* GFX9+ */
+ unsigned db_stencil_info2; /* GFX9 only */
unsigned db_htile_surface;
};
uint64_t debug_flags;
char renderer_string[183];
+ void (*make_texture_descriptor)(
+ struct si_screen *screen,
+ struct si_texture *tex,
+ bool sampler,
+ enum pipe_texture_target target,
+ enum pipe_format pipe_format,
+ const unsigned char state_swizzle[4],
+ unsigned first_level, unsigned last_level,
+ unsigned first_layer, unsigned last_layer,
+ unsigned width, unsigned height, unsigned depth,
+ uint32_t *state,
+ uint32_t *fmask_state);
+
unsigned pa_sc_raster_config;
unsigned pa_sc_raster_config_1;
unsigned se_tile_repeat;
unsigned eqaa_force_coverage_samples;
unsigned eqaa_force_z_samples;
unsigned eqaa_force_color_samples;
- bool has_clear_state;
- bool has_distributed_tess;
bool has_draw_indirect_multi;
bool has_out_of_order_rast;
bool assume_no_z_fights;
bool commutative_blend_add;
- bool clear_db_cache_before_clear;
- bool has_msaa_sample_loc_bug;
- bool has_ls_vgpr_init_bug;
- bool has_dcc_constant_encode;
bool dpbb_allowed;
bool dfsm_allowed;
bool llvm_has_working_vgpr_indexing;
+ bool use_ngg;
+ bool use_ngg_streamout;
+
+ struct {
+#define OPT_BOOL(name, dflt, description) bool name:1;
+#include "si_debug_options.h"
+ } options;
/* Whether shaders are monolithic (1-part) or separate (3-part). */
bool use_monolithic_shaders;
bool record_llvm_ir;
- bool has_rbplus; /* if RB+ registers exist */
- bool rbplus_allowed; /* if RB+ is allowed */
bool dcc_msaa_allowed;
- bool cpdma_prefetch_writes_memory;
struct slab_parent_pool pool_transfers;
/* Auxiliary context. Mainly used to initialize resources.
* It must be locked prior to using and flushed before unlocking. */
struct pipe_context *aux_context;
- mtx_t aux_context_lock;
+ simple_mtx_t aux_context_lock;
/* This must be in the screen, because UE4 uses one context for
* compilation and another one for rendering.
unsigned num_shader_cache_hits;
/* GPU load thread. */
- mtx_t gpu_load_mutex;
+ simple_mtx_t gpu_load_mutex;
thrd_t gpu_load_thread;
union si_mmio_counters mmio_counters;
volatile unsigned gpu_load_stop_thread; /* bool */
* the counter before drawing and re-emit the states accordingly.
*/
unsigned dirty_tex_counter;
+ unsigned dirty_buf_counter;
/* Atomically increment this counter when an existing texture's
* metadata is enabled or disabled in a way that requires changing
unsigned L2_to_cp;
} barrier_flags;
- mtx_t shader_parts_mutex;
+ simple_mtx_t shader_parts_mutex;
struct si_shader_part *vs_prologs;
struct si_shader_part *tcs_epilogs;
struct si_shader_part *gs_prologs;
* - GS and CS aren't cached, but it's certainly possible to cache
* those as well.
*/
- mtx_t shader_cache_mutex;
+ simple_mtx_t shader_cache_mutex;
struct hash_table *shader_cache;
/* Shader compiler queue for multithreaded compilation. */
/* Use at most 2 low priority threads on quadcore and better.
* We want to minimize the impact on multithreaded Mesa. */
struct ac_llvm_compiler compiler_lowp[10];
+
+ unsigned compute_wave_size;
+ unsigned ps_wave_size;
+ unsigned ge_wave_size;
};
struct si_blend_color {
#define SI_SAMPLER_STATE_MAGIC 0x34f1c35a
struct si_sampler_state {
-#ifdef DEBUG
+#ifndef NDEBUG
unsigned magic;
#endif
uint32_t val[4];
ubyte nr_color_samples; /* at most 8xAA */
ubyte compressed_cb_mask;
ubyte uncompressed_cb_mask;
+ ubyte displayable_dcc_cb_mask;
ubyte color_is_int8;
ubyte color_is_int10;
ubyte dirty_cbufs;
ubyte dcc_overwrite_combiner_watermark;
+ ubyte min_bytes_per_pixel;
bool dirty_zsbuf;
bool any_dst_linear;
bool CB_has_shader_readable_metadata;
bool DB_has_shader_readable_metadata;
+ bool all_DCC_pipe_aligned;
};
enum si_quant_mode {
enum si_quant_mode quant_mode;
};
-struct si_scissors {
- unsigned dirty_mask;
- struct pipe_scissor_state states[SI_MAX_VIEWPORTS];
-};
-
struct si_viewports {
- unsigned dirty_mask;
- unsigned depth_range_dirty_mask;
struct pipe_viewport_state states[SI_MAX_VIEWPORTS];
struct si_signed_scissor as_scissor[SI_MAX_VIEWPORTS];
+ bool y_inverted;
};
struct si_clip_state {
struct pipe_stream_output_target b;
/* The buffer where BUFFER_FILLED_SIZE is stored. */
- struct r600_resource *buf_filled_size;
+ struct si_resource *buf_filled_size;
unsigned buf_filled_size_offset;
bool buf_filled_size_valid;
*/
union si_vgt_param_key {
struct {
-#ifdef PIPE_ARCH_LITTLE_ENDIAN
+#if UTIL_ARCH_LITTLE_ENDIAN
unsigned prim:4;
unsigned uses_instancing:1;
unsigned multi_instances_smaller_than_primgroup:1;
unsigned tess_uses_prim_id:1;
unsigned uses_gs:1;
unsigned _pad:32 - SI_NUM_VGT_PARAM_KEY_BITS;
-#else /* PIPE_ARCH_BIG_ENDIAN */
+#else /* UTIL_ARCH_BIG_ENDIAN */
unsigned _pad:32 - SI_NUM_VGT_PARAM_KEY_BITS;
unsigned uses_gs:1;
unsigned tess_uses_prim_id:1;
uint32_t index;
};
+#define SI_NUM_VGT_STAGES_KEY_BITS 5
+#define SI_NUM_VGT_STAGES_STATES (1 << SI_NUM_VGT_STAGES_KEY_BITS)
+
+/* The VGT_SHADER_STAGES key used to index the table of precomputed values.
+ * Some fields are set by state-change calls, most are set by draw_vbo.
+ */
+union si_vgt_stages_key {
+ struct {
+#if UTIL_ARCH_LITTLE_ENDIAN
+ unsigned tess:1;
+ unsigned gs:1;
+ unsigned ngg_passthrough:1;
+ unsigned ngg:1; /* gfx10+ */
+ unsigned streamout:1; /* only used with NGG */
+ unsigned _pad:32 - SI_NUM_VGT_STAGES_KEY_BITS;
+#else /* UTIL_ARCH_BIG_ENDIAN */
+ unsigned _pad:32 - SI_NUM_VGT_STAGES_KEY_BITS;
+ unsigned streamout:1;
+ unsigned ngg:1;
+ unsigned ngg_passthrough:1;
+ unsigned gs:1;
+ unsigned tess:1;
+#endif
+ } u;
+ uint32_t index;
+};
+
struct si_texture_handle
{
unsigned desc_slot;
struct pipe_reference reference;
struct si_context *ctx;
struct radeon_saved_cs gfx;
- struct r600_resource *trace_buf;
+ struct radeon_saved_cs compute;
+ struct si_resource *trace_buf;
unsigned trace_id;
unsigned gfx_last_dw;
+ unsigned compute_last_dw;
bool flushed;
int64_t time_flush;
};
+struct si_sdma_upload {
+ struct si_resource *dst;
+ struct si_resource *src;
+ unsigned src_offset;
+ unsigned dst_offset;
+ unsigned size;
+};
+
struct si_context {
struct pipe_context b; /* base class */
struct radeon_winsys *ws;
struct radeon_winsys_ctx *ctx;
- struct radeon_cmdbuf *gfx_cs;
- struct radeon_cmdbuf *dma_cs;
+ struct radeon_cmdbuf *gfx_cs; /* compute IB if graphics is disabled */
+ struct radeon_cmdbuf *sdma_cs;
struct pipe_fence_handle *last_gfx_fence;
struct pipe_fence_handle *last_sdma_fence;
- struct r600_resource *eop_bug_scratch;
+ struct si_resource *eop_bug_scratch;
struct u_upload_mgr *cached_gtt_allocator;
struct threaded_context *tc;
struct u_suballocator *allocator_zeroed_memory;
struct pipe_device_reset_callback device_reset_callback;
struct u_log_context *log;
void *query_result_shader;
+ void *sh_query_result_shader;
+
+ void (*emit_cache_flush)(struct si_context *ctx);
+
struct blitter_context *blitter;
+ void *noop_blend;
+ void *noop_dsa;
+ void *discard_rasterizer_state;
void *custom_dsa_flush;
void *custom_blend_resolve;
void *custom_blend_fmask_decompress;
void *cs_copy_buffer;
void *cs_copy_image;
void *cs_copy_image_1d_array;
+ void *cs_clear_render_target;
+ void *cs_clear_render_target_1d_array;
+ void *cs_clear_12bytes_buffer;
+ void *cs_dcc_retile;
+ void *cs_fmask_expand[3][2]; /* [log2(samples)-1][is_array] */
struct si_screen *screen;
struct pipe_debug_callback debug;
struct ac_llvm_compiler compiler; /* only non-threaded compilation */
struct si_shader_ctx_state fixed_func_tcs_shader;
- struct r600_resource *wait_mem_scratch;
+ /* Offset 0: EOP flush number; Offset 4: GDS prim restart counter */
+ struct si_resource *wait_mem_scratch;
unsigned wait_mem_number;
uint16_t prefetch_L2_mask;
+ bool has_graphics;
bool gfx_flush_in_progress:1;
bool gfx_last_ib_is_busy:1;
bool compute_is_busy:1;
unsigned num_gfx_cs_flushes;
unsigned initial_gfx_cs_size;
- unsigned gpu_reset_counter;
unsigned last_dirty_tex_counter;
+ unsigned last_dirty_buf_counter;
unsigned last_compressed_colortex_counter;
unsigned last_num_draw_calls;
unsigned flags; /* flush flags */
uint64_t vram;
uint64_t gtt;
+ /* Compute-based primitive discard. */
+ unsigned prim_discard_vertex_count_threshold;
+ struct pb_buffer *gds;
+ struct pb_buffer *gds_oa;
+ struct radeon_cmdbuf *prim_discard_compute_cs;
+ unsigned compute_gds_offset;
+ struct si_shader *compute_ib_last_shader;
+ uint32_t compute_rewind_va;
+ unsigned compute_num_prims_in_batch;
+ bool preserve_prim_restart_gds_at_flush;
+ /* index_ring is divided into 2 halves for doublebuffering. */
+ struct si_resource *index_ring;
+ unsigned index_ring_base; /* offset of a per-IB portion */
+ unsigned index_ring_offset; /* offset within a per-IB portion */
+ unsigned index_ring_size_per_ib; /* max available size per IB */
+ bool prim_discard_compute_ib_initialized;
+ /* For tracking the last execution barrier - it can be either
+ * a WRITE_DATA packet or a fence. */
+ uint32_t *last_pkt3_write_data;
+ struct si_resource *barrier_buf;
+ unsigned barrier_buf_offset;
+ struct pipe_fence_handle *last_ib_barrier_fence;
+ struct si_resource *last_ib_barrier_buf;
+ unsigned last_ib_barrier_buf_offset;
+
/* Atoms (direct states). */
union si_state_atoms atoms;
unsigned dirty_atoms; /* mask */
struct si_clip_state clip_state;
struct si_shader_data shader_pointers;
struct si_stencil_ref stencil_ref;
- struct si_scissors scissors;
+ struct pipe_scissor_state scissors[SI_MAX_VIEWPORTS];
struct si_streamout streamout;
struct si_viewports viewports;
unsigned num_window_rectangles;
struct si_pm4_state *init_config;
struct si_pm4_state *init_config_gs_rings;
bool init_config_has_vgt_flush;
- struct si_pm4_state *vgt_shader_config[4];
+ struct si_pm4_state *vgt_shader_config[SI_NUM_VGT_STAGES_STATES];
/* shaders */
struct si_shader_ctx_state ps_shader;
struct si_shader_ctx_state vs_shader;
struct si_shader_ctx_state tcs_shader;
struct si_shader_ctx_state tes_shader;
+ struct si_shader_ctx_state cs_prim_discard_state;
struct si_cs_shader_state cs_shader_state;
/* shader information */
/* vertex buffer descriptors */
uint32_t *vb_descriptors_gpu_list;
- struct r600_resource *vb_descriptors_buffer;
+ struct si_resource *vb_descriptors_buffer;
unsigned vb_descriptors_offset;
/* shader descriptors */
struct si_buffer_resources const_and_shader_buffers[SI_NUM_SHADERS];
struct si_samplers samplers[SI_NUM_SHADERS];
struct si_images images[SI_NUM_SHADERS];
+ bool bo_list_add_all_resident_resources;
+ bool bo_list_add_all_gfx_resources;
+ bool bo_list_add_all_compute_resources;
/* other shader resources */
- struct pipe_constant_buffer null_const_buf; /* used for set_constant_buffer(NULL) on CIK */
+ struct pipe_constant_buffer null_const_buf; /* used for set_constant_buffer(NULL) on GFX7 */
struct pipe_resource *esgs_ring;
struct pipe_resource *gsvs_ring;
struct pipe_resource *tess_rings;
union pipe_color_union *border_color_table; /* in CPU memory, any endian */
- struct r600_resource *border_color_buffer;
+ struct si_resource *border_color_buffer;
union pipe_color_union *border_color_map; /* in VRAM (slow access), little endian */
unsigned border_color_count;
unsigned num_vs_blit_sgprs;
uint32_t vs_blit_sh_data[SI_VS_BLIT_SGPRS_POS_TEXCOORD];
uint32_t cs_user_data[4];
- /**
- * last_block allows disabling threads at the farthermost grid boundary.
- * Full blocks as specified by "block" are launched, but the threads
- * outside of "last_block" dimensions are disabled.
- *
- * If a block touches the grid boundary in the i-th axis, threads with
- * THREAD_ID[i] >= last_block[i] are disabled.
- *
- * If last_block[i] is 0, it has the same behavior as last_block[i] = block[i],
- * meaning no effect.
- *
- * It's equivalent to doing this at the beginning of the compute shader:
- *
- * for (i = 0; i < 3; i++) {
- * if (block_id[i] == grid[i] - 1 &&
- * last_block[i] && last_block[i] >= thread_id[i])
- * return;
- * }
- * (this could be moved into pipe_grid_info)
- */
- uint compute_last_block[3];
-
/* Vertex and index buffers. */
bool vertex_buffers_dirty;
bool vertex_buffer_pointer_dirty;
struct pipe_vertex_buffer vertex_buffer[SI_NUM_VERTEX_BUFFERS];
+ uint16_t vertex_buffer_unaligned; /* bitmask of not dword-aligned buffers */
/* MSAA config state. */
int ps_iter_samples;
/* Emitted draw state. */
bool gs_tri_strip_adj_fix:1;
bool ls_vgpr_fix:1;
+ bool prim_discard_cs_instancing:1;
+ bool ngg:1;
int last_index_size;
int last_base_vertex;
int last_start_instance;
int last_prim;
int last_multi_vgt_param;
int last_rast_prim;
+ int last_flatshade_first;
+ int last_binning_enabled;
unsigned last_sc_line_stipple;
unsigned current_vs_state;
unsigned last_vs_state;
enum pipe_prim_type current_rast_prim; /* primitive type after TES, GS */
/* Scratch buffer */
- struct r600_resource *scratch_buffer;
+ struct si_resource *scratch_buffer;
unsigned scratch_waves;
unsigned spi_tmpring_size;
+ unsigned max_seen_scratch_bytes_per_wave;
+ unsigned max_seen_compute_scratch_bytes_per_wave;
- struct r600_resource *compute_scratch_buffer;
+ struct si_resource *compute_scratch_buffer;
/* Emitted derived tessellation state. */
/* Local shader (VS), or HS if LS-HS are merged. */
unsigned num_resident_handles;
uint64_t num_alloc_tex_transfer_bytes;
unsigned last_tex_ps_draw_ratio; /* for query */
- unsigned context_roll_counter;
+ unsigned compute_num_verts_accepted;
+ unsigned compute_num_verts_rejected;
+ unsigned compute_num_verts_ineligible; /* due to low vertex count */
+ unsigned context_roll;
/* Queries. */
/* Maintain the list of active queries for pausing between IBs. */
int num_occlusion_queries;
int num_perfect_occlusion_queries;
+ int num_pipeline_stat_queries;
struct list_head active_queries;
unsigned num_cs_dw_queries_suspend;
bool render_cond_invert;
bool render_cond_force_off; /* for u_blitter */
+ /* For uploading data via GTT and copy to VRAM on context flush via SDMA. */
+ bool sdma_uploads_in_progress;
+ struct si_sdma_upload *sdma_uploads;
+ unsigned num_sdma_uploads;
+ unsigned max_sdma_uploads;
+
+ /* Shader-based queries. */
+ struct list_head shader_query_buffers;
+ unsigned num_active_shader_queries;
+
/* Statistics gathering for the DCC enablement heuristic. It can't be
* in si_texture because si_texture can be shared by multiple
* contexts. This is for back buffers only. We shouldn't get too many
unsigned src_level,
const struct pipe_box *src_box);
void si_decompress_dcc(struct si_context *sctx, struct si_texture *tex);
-void si_blit_decompress_depth(struct pipe_context *ctx,
- struct si_texture *texture,
- struct si_texture *staging,
- unsigned first_level, unsigned last_level,
- unsigned first_layer, unsigned last_layer,
- unsigned first_sample, unsigned last_sample);
/* si_buffer.c */
bool si_rings_is_buffer_referenced(struct si_context *sctx,
struct pb_buffer *buf,
enum radeon_bo_usage usage);
void *si_buffer_map_sync_with_rings(struct si_context *sctx,
- struct r600_resource *resource,
+ struct si_resource *resource,
unsigned usage);
void si_init_resource_fields(struct si_screen *sscreen,
- struct r600_resource *res,
+ struct si_resource *res,
uint64_t size, unsigned alignment);
bool si_alloc_resource(struct si_screen *sscreen,
- struct r600_resource *res);
+ struct si_resource *res);
struct pipe_resource *pipe_aligned_buffer_create(struct pipe_screen *screen,
unsigned flags, unsigned usage,
unsigned size, unsigned alignment);
-struct r600_resource *si_aligned_buffer_create(struct pipe_screen *screen,
+struct si_resource *si_aligned_buffer_create(struct pipe_screen *screen,
unsigned flags, unsigned usage,
unsigned size, unsigned alignment);
void si_replace_buffer_storage(struct pipe_context *ctx,
/* si_clear.c */
enum pipe_format si_simplify_cb_format(enum pipe_format format);
-bool vi_alpha_is_on_msb(enum pipe_format format);
-void vi_dcc_clear_level(struct si_context *sctx,
+bool vi_alpha_is_on_msb(struct si_screen *sscreen, enum pipe_format format);
+bool vi_dcc_clear_level(struct si_context *sctx,
struct si_texture *tex,
unsigned level, unsigned clear_value);
void si_init_clear_functions(struct si_context *sctx);
enum si_cache_policy cache_policy);
void si_clear_buffer(struct si_context *sctx, struct pipe_resource *dst,
uint64_t offset, uint64_t size, uint32_t *clear_value,
- uint32_t clear_value_size, enum si_coherency coher);
+ uint32_t clear_value_size, enum si_coherency coher,
+ bool force_cpdma);
void si_copy_buffer(struct si_context *sctx,
struct pipe_resource *dst, struct pipe_resource *src,
uint64_t dst_offset, uint64_t src_offset, unsigned size);
unsigned src_level,
unsigned dstx, unsigned dsty, unsigned dstz,
const struct pipe_box *src_box);
+void si_compute_clear_render_target(struct pipe_context *ctx,
+ struct pipe_surface *dstsurf,
+ const union pipe_color_union *color,
+ unsigned dstx, unsigned dsty,
+ unsigned width, unsigned height,
+ bool render_condition_enabled);
+void si_retile_dcc(struct si_context *sctx, struct si_texture *tex);
+void si_compute_expand_fmask(struct pipe_context *ctx, struct pipe_resource *tex);
void si_init_compute_blit_functions(struct si_context *sctx);
/* si_cp_dma.c */
uint64_t offset, unsigned size);
void cik_emit_prefetch_L2(struct si_context *sctx, bool vertex_stage_only);
void si_test_gds(struct si_context *sctx);
-void si_cp_write_data(struct si_context *sctx, struct r600_resource *buf,
+void si_cp_write_data(struct si_context *sctx, struct si_resource *buf,
unsigned offset, unsigned size, unsigned dst_sel,
unsigned engine, const void *data);
+void si_cp_copy_data(struct si_context *sctx, struct radeon_cmdbuf *cs,
+ unsigned dst_sel, struct si_resource *dst, unsigned dst_offset,
+ unsigned src_sel, struct si_resource *src, unsigned src_offset);
/* si_debug.c */
void si_save_cs(struct radeon_winsys *ws, struct radeon_cmdbuf *cs,
void si_init_debug_functions(struct si_context *sctx);
void si_check_vm_faults(struct si_context *sctx,
struct radeon_saved_cs *saved, enum ring_type ring);
-bool si_replace_shader(unsigned num, struct ac_shader_binary *binary);
+bool si_replace_shader(unsigned num, struct si_shader_binary *binary);
/* si_dma.c */
void si_init_dma_functions(struct si_context *sctx);
/* si_dma_cs.c */
-void si_dma_emit_timestamp(struct si_context *sctx, struct r600_resource *dst,
+void si_dma_emit_timestamp(struct si_context *sctx, struct si_resource *dst,
uint64_t offset);
void si_sdma_clear_buffer(struct si_context *sctx, struct pipe_resource *dst,
uint64_t offset, uint64_t size, unsigned clear_value);
void si_need_dma_space(struct si_context *ctx, unsigned num_dw,
- struct r600_resource *dst, struct r600_resource *src);
+ struct si_resource *dst, struct si_resource *src);
void si_flush_dma_cs(struct si_context *ctx, unsigned flags,
struct pipe_fence_handle **fence);
void si_screen_clear_buffer(struct si_screen *sscreen, struct pipe_resource *dst,
uint64_t offset, uint64_t size, unsigned value);
/* si_fence.c */
-void si_cp_release_mem(struct si_context *ctx,
+void si_cp_release_mem(struct si_context *ctx, struct radeon_cmdbuf *cs,
unsigned event, unsigned event_flags,
unsigned dst_sel, unsigned int_sel, unsigned data_sel,
- struct r600_resource *buf, uint64_t va,
+ struct si_resource *buf, uint64_t va,
uint32_t new_fence, unsigned query_type);
unsigned si_cp_write_fence_dwords(struct si_screen *screen);
void si_cp_wait_mem(struct si_context *ctx, struct radeon_cmdbuf *cs,
/* si_gfx_cs.c */
void si_flush_gfx_cs(struct si_context *ctx, unsigned flags,
struct pipe_fence_handle **fence);
+void si_allocate_gds(struct si_context *ctx);
void si_begin_new_gfx_cs(struct si_context *ctx);
void si_need_gfx_cs_space(struct si_context *ctx);
+void si_unref_sdma_uploads(struct si_context *sctx);
/* si_gpu_load.c */
void si_gpu_load_kill_thread(struct si_screen *sscreen);
uint64_t begin);
/* si_compute.c */
+void si_emit_initial_compute_regs(struct si_context *sctx, struct radeon_cmdbuf *cs);
void si_init_compute_functions(struct si_context *sctx);
+/* si_compute_prim_discard.c */
+enum si_prim_discard_outcome {
+ SI_PRIM_DISCARD_ENABLED,
+ SI_PRIM_DISCARD_DISABLED,
+ SI_PRIM_DISCARD_DRAW_SPLIT,
+};
+
+void si_build_prim_discard_compute_shader(struct si_shader_context *ctx);
+enum si_prim_discard_outcome
+si_prepare_prim_discard_or_split_draw(struct si_context *sctx,
+ const struct pipe_draw_info *info,
+ bool primitive_restart);
+void si_compute_signal_gfx(struct si_context *sctx);
+void si_dispatch_prim_discard_cs_and_draw(struct si_context *sctx,
+ const struct pipe_draw_info *info,
+ unsigned index_size,
+ unsigned base_vertex,
+ uint64_t input_indexbuf_va,
+ unsigned input_indexbuf_max_elements);
+void si_initialize_prim_discard_tunables(struct si_context *sctx);
+
+/* si_pipe.c */
+void si_init_compiler(struct si_screen *sscreen, struct ac_llvm_compiler *compiler);
+
/* si_perfcounters.c */
void si_init_perfcounters(struct si_screen *screen);
void si_destroy_perfcounters(struct si_screen *screen);
-/* si_pipe.c */
-bool si_check_device_reset(struct si_context *sctx);
-
/* si_query.c */
void si_init_screen_query_functions(struct si_screen *sscreen);
void si_init_query_functions(struct si_context *sctx);
bool dst_stream_cache_policy, bool is_copy);
void *si_create_copy_image_compute_shader(struct pipe_context *ctx);
void *si_create_copy_image_compute_shader_1d_array(struct pipe_context *ctx);
+void *si_clear_render_target_shader(struct pipe_context *ctx);
+void *si_clear_render_target_shader_1d_array(struct pipe_context *ctx);
+void *si_clear_12bytes_buffer_shader(struct pipe_context *ctx);
+void *si_create_dcc_retile_cs(struct pipe_context *ctx);
+void *si_create_fmask_expand_cs(struct pipe_context *ctx, unsigned num_samples,
+ bool is_array);
void *si_create_query_result_cs(struct si_context *sctx);
+void *gfx10_create_sh_query_result_cs(struct si_context *sctx);
+
+/* gfx10_query.c */
+void gfx10_init_query(struct si_context *sctx);
+void gfx10_destroy_query(struct si_context *sctx);
/* si_test_dma.c */
void si_test_dma(struct si_screen *sscreen);
void si_texture_discard_cmask(struct si_screen *sscreen,
struct si_texture *tex);
bool si_init_flushed_depth_texture(struct pipe_context *ctx,
- struct pipe_resource *texture,
- struct si_texture **staging);
+ struct pipe_resource *texture);
void si_print_texture_info(struct si_screen *sscreen,
struct si_texture *tex, struct u_log_context *log);
struct pipe_resource *si_texture_create(struct pipe_screen *screen,
const struct pipe_resource *templ);
-bool vi_dcc_formats_compatible(enum pipe_format format1,
+bool vi_dcc_formats_compatible(struct si_screen *sscreen,
+ enum pipe_format format1,
enum pipe_format format2);
bool vi_dcc_formats_are_incompatible(struct pipe_resource *tex,
unsigned level,
* common helpers
*/
-static inline struct r600_resource *r600_resource(struct pipe_resource *r)
+static inline struct si_resource *si_resource(struct pipe_resource *r)
{
- return (struct r600_resource*)r;
+ return (struct si_resource*)r;
}
static inline void
-r600_resource_reference(struct r600_resource **ptr, struct r600_resource *res)
+si_resource_reference(struct si_resource **ptr, struct si_resource *res)
{
pipe_resource_reference((struct pipe_resource **)ptr,
(struct pipe_resource *)res);
static inline bool
vi_dcc_enabled(struct si_texture *tex, unsigned level)
{
- return tex->dcc_offset && level < tex->surface.num_dcc_levels;
+ return tex->surface.dcc_offset && level < tex->surface.num_dcc_levels;
}
static inline unsigned
return tex->surface.u.legacy.tiling_index[level];
}
+static inline unsigned
+si_get_minimum_num_gfx_cs_dwords(struct si_context *sctx)
+{
+ /* Don't count the needed CS space exactly and just use an upper bound.
+ *
+ * Also reserve space for stopping queries at the end of IB, because
+ * the number of active queries is unlimited in theory.
+ */
+ return 2048 + sctx->num_cs_dw_queries_suspend;
+}
+
static inline void
si_context_add_resource_size(struct si_context *sctx, struct pipe_resource *r)
{
if (r) {
/* Add memory usage for need_gfx_cs_space */
- sctx->vram += r600_resource(r)->vram_usage;
- sctx->gtt += r600_resource(r)->gart_usage;
+ sctx->vram += si_resource(r)->vram_usage;
+ sctx->gtt += si_resource(r)->gart_usage;
}
}
static inline struct si_shader* si_get_vs_state(struct si_context *sctx)
{
- if (sctx->gs_shader.cso)
+ if (sctx->gs_shader.cso &&
+ sctx->gs_shader.current &&
+ !sctx->gs_shader.current->key.as_ngg)
return sctx->gs_shader.cso->gs_copy_shader;
struct si_shader_ctx_state *vs = si_get_vs(sctx);
static inline void
si_make_CB_shader_coherent(struct si_context *sctx, unsigned num_samples,
- bool shaders_read_metadata)
+ bool shaders_read_metadata, bool dcc_pipe_aligned)
{
sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_CB |
- SI_CONTEXT_INV_VMEM_L1;
+ SI_CONTEXT_INV_VCACHE;
- if (sctx->chip_class >= GFX9) {
+ if (sctx->chip_class >= GFX10) {
+ if (sctx->screen->info.tcc_harvested)
+ sctx->flags |= SI_CONTEXT_INV_L2;
+ else if (shaders_read_metadata)
+ sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
+ } else if (sctx->chip_class == GFX9) {
/* Single-sample color is coherent with shaders on GFX9, but
* L2 metadata must be flushed if shaders read metadata.
* (DCC, CMASK).
*/
- if (num_samples >= 2)
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ if (num_samples >= 2 ||
+ (shaders_read_metadata && !dcc_pipe_aligned))
+ sctx->flags |= SI_CONTEXT_INV_L2;
else if (shaders_read_metadata)
sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
} else {
- /* SI-CI-VI */
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ /* GFX6-GFX8 */
+ sctx->flags |= SI_CONTEXT_INV_L2;
}
}
bool include_stencil, bool shaders_read_metadata)
{
sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_DB |
- SI_CONTEXT_INV_VMEM_L1;
+ SI_CONTEXT_INV_VCACHE;
- if (sctx->chip_class >= GFX9) {
+ if (sctx->chip_class >= GFX10) {
+ if (sctx->screen->info.tcc_harvested)
+ sctx->flags |= SI_CONTEXT_INV_L2;
+ else if (shaders_read_metadata)
+ sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
+ } else if (sctx->chip_class == GFX9) {
/* Single-sample depth (not stencil) is coherent with shaders
* on GFX9, but L2 metadata must be flushed if shaders read
* metadata.
*/
if (num_samples >= 2 || include_stencil)
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ sctx->flags |= SI_CONTEXT_INV_L2;
else if (shaders_read_metadata)
sctx->flags |= SI_CONTEXT_INV_L2_METADATA;
} else {
- /* SI-CI-VI */
- sctx->flags |= SI_CONTEXT_INV_GLOBAL_L2;
+ /* GFX6-GFX8 */
+ sctx->flags |= SI_CONTEXT_INV_L2;
}
}
}
static inline bool
-si_htile_enabled(struct si_texture *tex, unsigned level)
+si_htile_enabled(struct si_texture *tex, unsigned level, unsigned zs_mask)
{
- return tex->htile_offset && level == 0;
+ if (zs_mask == PIPE_MASK_S && tex->htile_stencil_disabled)
+ return false;
+
+ return tex->surface.htile_offset && level == 0;
}
static inline bool
-vi_tc_compat_htile_enabled(struct si_texture *tex, unsigned level)
+vi_tc_compat_htile_enabled(struct si_texture *tex, unsigned level, unsigned zs_mask)
{
- assert(!tex->tc_compatible_htile || tex->htile_offset);
- return tex->tc_compatible_htile && level == 0;
+ assert(!tex->tc_compatible_htile || tex->surface.htile_offset);
+ return tex->tc_compatible_htile && si_htile_enabled(tex, level, zs_mask);
}
static inline unsigned si_get_ps_iter_samples(struct si_context *sctx)
(1 << PIPE_PRIM_POINTS))) != 0;
}
+static inline bool util_rast_prim_is_triangles(unsigned prim)
+{
+ return ((1 << prim) & ((1 << PIPE_PRIM_TRIANGLES) |
+ (1 << PIPE_PRIM_TRIANGLE_STRIP) |
+ (1 << PIPE_PRIM_TRIANGLE_FAN) |
+ (1 << PIPE_PRIM_QUADS) |
+ (1 << PIPE_PRIM_QUAD_STRIP) |
+ (1 << PIPE_PRIM_POLYGON) |
+ (1 << PIPE_PRIM_TRIANGLES_ADJACENCY) |
+ (1 << PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY)));
+}
+
/**
* Return true if there is enough memory in VRAM and GTT for the buffers
* added so far.
*/
static inline void radeon_add_to_buffer_list(struct si_context *sctx,
struct radeon_cmdbuf *cs,
- struct r600_resource *rbo,
+ struct si_resource *bo,
enum radeon_bo_usage usage,
enum radeon_bo_priority priority)
{
assert(usage);
sctx->ws->cs_add_buffer(
- cs, rbo->buf,
+ cs, bo->buf,
(enum radeon_bo_usage)(usage | RADEON_USAGE_SYNCHRONIZED),
- rbo->domains, priority);
+ bo->domains, priority);
}
/**
*/
static inline void
radeon_add_to_gfx_buffer_list_check_mem(struct si_context *sctx,
- struct r600_resource *rbo,
+ struct si_resource *bo,
enum radeon_bo_usage usage,
enum radeon_bo_priority priority,
bool check_mem)
{
if (check_mem &&
!radeon_cs_memory_below_limit(sctx->screen, sctx->gfx_cs,
- sctx->vram + rbo->vram_usage,
- sctx->gtt + rbo->gart_usage))
+ sctx->vram + bo->vram_usage,
+ sctx->gtt + bo->gart_usage))
si_flush_gfx_cs(sctx, RADEON_FLUSH_ASYNC_START_NEXT_GFX_IB_NOW, NULL);
- radeon_add_to_buffer_list(sctx, sctx->gfx_cs, rbo, usage, priority);
+ radeon_add_to_buffer_list(sctx, sctx->gfx_cs, bo, usage, priority);
+}
+
+static inline bool si_compute_prim_discard_enabled(struct si_context *sctx)
+{
+ return sctx->prim_discard_vertex_count_threshold != UINT_MAX;
+}
+
+static inline unsigned si_get_wave_size(struct si_screen *sscreen,
+ enum pipe_shader_type shader_type,
+ bool ngg, bool es)
+{
+ if (shader_type == PIPE_SHADER_COMPUTE)
+ return sscreen->compute_wave_size;
+ else if (shader_type == PIPE_SHADER_FRAGMENT)
+ return sscreen->ps_wave_size;
+ else if ((shader_type == PIPE_SHADER_VERTEX && es && !ngg) ||
+ (shader_type == PIPE_SHADER_TESS_EVAL && es && !ngg) ||
+ (shader_type == PIPE_SHADER_GEOMETRY && !ngg)) /* legacy GS only supports Wave64 */
+ return 64;
+ else
+ return sscreen->ge_wave_size;
+}
+
+static inline unsigned si_get_shader_wave_size(struct si_shader *shader)
+{
+ return si_get_wave_size(shader->selector->screen, shader->selector->type,
+ shader->key.as_ngg, shader->key.as_es);
}
#define PRINT_ERR(fmt, args...) \
projects / mesa.git / blobdiff