#include <llvm-c/Core.h> /* LLVMModuleRef */
#include <llvm-c/TargetMachine.h>
-#include "tgsi/tgsi_scan.h"
#include "util/u_inlines.h"
#include "util/u_queue.h"
+#include "util/simple_mtx.h"
#include "ac_binary.h"
#include "ac_llvm_build.h"
/* Shader IO unique indices are supported for TGSI_SEMANTIC_GENERIC with an
* index smaller than this.
*/
-#define SI_MAX_IO_GENERIC 43
+#define SI_MAX_IO_GENERIC 32
/* SGPR user data indices */
enum {
/* PS only */
SI_SGPR_ALPHA_REF = SI_NUM_RESOURCE_SGPRS,
SI_PS_NUM_USER_SGPR,
+
+ /* The value has to be 12, because the hw requires that descriptors
+ * are aligned to 4 SGPRs.
+ */
+ SI_SGPR_VS_VB_DESCRIPTOR_FIRST = 12,
};
/* LLVM function parameter indices */
};
/* Fields of driver-defined VS state SGPR. */
-/* Clamp vertex color output (only used in VS as VS). */
#define S_VS_STATE_CLAMP_VERTEX_COLOR(x) (((unsigned)(x) & 0x1) << 0)
#define C_VS_STATE_CLAMP_VERTEX_COLOR 0xFFFFFFFE
#define S_VS_STATE_INDEXED(x) (((unsigned)(x) & 0x1) << 1)
#define C_VS_STATE_INDEXED 0xFFFFFFFD
+#define S_VS_STATE_OUTPRIM(x) (((unsigned)(x) & 0x3) << 2)
+#define C_VS_STATE_OUTPRIM 0xFFFFFFF3
+#define S_VS_STATE_PROVOKING_VTX_INDEX(x) (((unsigned)(x) & 0x3) << 4)
+#define C_VS_STATE_PROVOKING_VTX_INDEX 0xFFFFFFCF
+#define S_VS_STATE_STREAMOUT_QUERY_ENABLED(x) (((unsigned)(x) & 0x1) << 6)
+#define C_VS_STATE_STREAMOUT_QUERY_ENABLED 0xFFFFFFBF
#define S_VS_STATE_LS_OUT_PATCH_SIZE(x) (((unsigned)(x) & 0x1FFF) << 8)
#define C_VS_STATE_LS_OUT_PATCH_SIZE 0xFFE000FF
#define S_VS_STATE_LS_OUT_VERTEX_SIZE(x) (((unsigned)(x) & 0xFF) << 24)
#define C_VS_STATE_LS_OUT_VERTEX_SIZE 0x00FFFFFF
-/* Driver-specific system values. */
-enum {
- /* Values from set_tess_state. */
- TGSI_SEMANTIC_DEFAULT_TESSOUTER_SI = TGSI_SEMANTIC_COUNT,
- TGSI_SEMANTIC_DEFAULT_TESSINNER_SI,
-
- /* Up to 4 dwords in user SGPRs for compute shaders. */
- TGSI_SEMANTIC_CS_USER_DATA,
-};
-
enum {
/* Use a property enum that CS wouldn't use. */
TGSI_PROPERTY_CS_LOCAL_SIZE = TGSI_PROPERTY_FS_COORD_ORIGIN,
- /* The number of used user data dwords in the range [1, 4]. */
- TGSI_PROPERTY_CS_USER_DATA_DWORDS = TGSI_PROPERTY_FS_COORD_PIXEL_CENTER,
-
- /* Use a property enum that VS wouldn't use. */
- TGSI_PROPERTY_VS_BLIT_SGPRS = TGSI_PROPERTY_FS_COORD_ORIGIN,
-
/* These represent the number of SGPRs the shader uses. */
SI_VS_BLIT_SGPRS_POS = 3,
SI_VS_BLIT_SGPRS_POS_COLOR = 7,
bool is_debug_context;
};
+struct si_shader_info {
+ ubyte num_inputs;
+ ubyte num_outputs;
+ ubyte input_semantic_name[PIPE_MAX_SHADER_INPUTS]; /**< TGSI_SEMANTIC_x */
+ ubyte input_semantic_index[PIPE_MAX_SHADER_INPUTS];
+ ubyte input_interpolate[PIPE_MAX_SHADER_INPUTS];
+ ubyte input_interpolate_loc[PIPE_MAX_SHADER_INPUTS];
+ ubyte input_usage_mask[PIPE_MAX_SHADER_INPUTS];
+ ubyte output_semantic_name[PIPE_MAX_SHADER_OUTPUTS]; /**< TGSI_SEMANTIC_x */
+ ubyte output_semantic_index[PIPE_MAX_SHADER_OUTPUTS];
+ ubyte output_usagemask[PIPE_MAX_SHADER_OUTPUTS];
+ ubyte output_streams[PIPE_MAX_SHADER_OUTPUTS];
+
+ ubyte processor;
+
+ int constbuf0_num_slots;
+ unsigned const_buffers_declared; /**< bitmask of declared const buffers */
+ unsigned samplers_declared; /**< bitmask of declared samplers */
+ ubyte num_stream_output_components[4];
+
+ uint num_memory_instructions; /**< sampler, buffer, and image instructions */
+
+ /**
+ * If a tessellation control shader reads outputs, this describes which ones.
+ */
+ bool reads_pervertex_outputs;
+ bool reads_perpatch_outputs;
+ bool reads_tessfactor_outputs;
+
+ ubyte colors_read; /**< which color components are read by the FS */
+ ubyte colors_written;
+ bool reads_samplemask; /**< does fragment shader read sample mask? */
+ bool reads_tess_factors; /**< If TES reads TESSINNER or TESSOUTER */
+ bool writes_z; /**< does fragment shader write Z value? */
+ bool writes_stencil; /**< does fragment shader write stencil value? */
+ bool writes_samplemask; /**< does fragment shader write sample mask? */
+ bool writes_edgeflag; /**< vertex shader outputs edgeflag */
+ bool uses_kill; /**< KILL or KILL_IF instruction used? */
+ bool uses_persp_center;
+ bool uses_persp_centroid;
+ bool uses_persp_sample;
+ bool uses_linear_center;
+ bool uses_linear_centroid;
+ bool uses_linear_sample;
+ bool uses_persp_opcode_interp_sample;
+ bool uses_linear_opcode_interp_sample;
+ bool uses_instanceid;
+ bool uses_vertexid;
+ bool uses_vertexid_nobase;
+ bool uses_basevertex;
+ bool uses_drawid;
+ bool uses_primid;
+ bool uses_frontface;
+ bool uses_invocationid;
+ bool uses_thread_id[3];
+ bool uses_block_id[3];
+ bool uses_block_size;
+ bool uses_grid_size;
+ bool uses_subgroup_info;
+ bool writes_position;
+ bool writes_psize;
+ bool writes_clipvertex;
+ bool writes_primid;
+ bool writes_viewport_index;
+ bool writes_layer;
+ bool writes_memory; /**< contains stores or atomics to buffers or images */
+ bool uses_derivatives;
+ bool uses_bindless_samplers;
+ bool uses_bindless_images;
+ bool uses_fbfetch;
+ unsigned clipdist_writemask;
+ unsigned culldist_writemask;
+ unsigned num_written_culldistance;
+ unsigned num_written_clipdistance;
+
+ unsigned images_declared; /**< bitmask of declared images */
+ unsigned msaa_images_declared; /**< bitmask of declared MSAA images */
+ unsigned shader_buffers_declared; /**< bitmask of declared shader buffers */
+
+ unsigned properties[TGSI_PROPERTY_COUNT]; /* index with TGSI_PROPERTY_ */
+
+ /** Whether all codepaths write tess factors in all invocations. */
+ bool tessfactors_are_def_in_all_invocs;
+};
+
/* A shader selector is a gallium CSO and contains shader variants and
- * binaries for one TGSI program. This can be shared by multiple contexts.
+ * binaries for one NIR program. This can be shared by multiple contexts.
*/
struct si_shader_selector {
struct pipe_reference reference;
struct util_queue_fence ready;
struct si_compiler_ctx_state compiler_ctx_state;
- mtx_t mutex;
+ simple_mtx_t mutex;
struct si_shader *first_variant; /* immutable after the first variant */
struct si_shader *last_variant; /* mutable */
- /* The compiled TGSI shader expecting a prolog and/or epilog (not
- * uploaded to a buffer).
+ /* The compiled NIR shader without a prolog and/or epilog (not
+ * uploaded to a buffer object).
*/
struct si_shader *main_shader_part;
struct si_shader *main_shader_part_ls; /* as_ls is set in the key */
struct si_shader *main_shader_part_es; /* as_es is set in the key */
struct si_shader *main_shader_part_ngg; /* as_ngg is set in the key */
+ struct si_shader *main_shader_part_ngg_es; /* for Wave32 TES before legacy GS */
struct si_shader *gs_copy_shader;
- struct tgsi_token *tokens;
struct nir_shader *nir;
+ void *nir_binary;
+ unsigned nir_size;
+
struct pipe_stream_output_info so;
- struct tgsi_shader_info info;
- struct tgsi_tessctrl_info tcs_info;
+ struct si_shader_info info;
/* PIPE_SHADER_[VERTEX|FRAGMENT|...] */
- unsigned type;
+ enum pipe_shader_type type;
bool vs_needs_prolog;
bool force_correct_derivs_after_kill;
bool prim_discard_cs_allowed;
+ unsigned num_vs_inputs;
+ unsigned num_vbos_in_user_sgprs;
unsigned pa_cl_vs_out_cntl;
ubyte clipdist_mask;
ubyte culldist_mask;
+ unsigned rast_prim;
/* ES parameters. */
unsigned esgs_itemsize; /* vertex stride */
unsigned gsvs_vertex_size;
unsigned max_gsvs_emit_size;
unsigned enabled_streamout_buffer_mask;
+ bool tess_turns_off_ngg;
/* PS parameters. */
unsigned color_attr_index[2];
unsigned num_input_sgprs:6;
/* For merged stages such as LS-HS, HS input VGPRs are first. */
unsigned num_merged_next_stage_vgprs:3;
- unsigned last_input:4;
+ unsigned num_inputs:5;
unsigned as_ls:1;
unsigned as_es:1;
unsigned as_ngg:1;
struct si_gs_prolog_bits states;
/* Prologs of monolithic shaders shouldn't set EXEC. */
unsigned is_monolithic:1;
+ unsigned as_ngg:1;
} gs_prolog;
struct {
struct si_ps_prolog_bits states;
unsigned vs_export_prim_id:1;
struct {
unsigned interpolate_at_sample_force_center:1;
- unsigned fbfetch_msaa;
- unsigned fbfetch_is_1D;
- unsigned fbfetch_layered;
+ unsigned fbfetch_msaa:1;
+ unsigned fbfetch_is_1D:1;
+ unsigned fbfetch_layered:1;
} ps;
} u;
} mono;
#pragma pack(pop)
/* GCN-specific shader info. */
-struct si_shader_info {
+struct si_shader_binary_info {
ubyte vs_output_param_offset[SI_MAX_VS_OUTPUTS];
ubyte num_input_sgprs;
ubyte num_input_vgprs;
/* The following data is all that's needed for binary shaders. */
struct si_shader_binary binary;
struct ac_shader_config config;
- struct si_shader_info info;
+ struct si_shader_binary_info info;
+
+ struct {
+ uint16_t ngg_emit_size; /* in dwords */
+ uint16_t hw_max_esverts;
+ uint16_t max_gsprims;
+ uint16_t max_out_verts;
+ uint16_t prim_amp_factor;
+ bool max_vert_out_per_gs_instance;
+ } ngg;
/* Shader key + LLVM IR + disassembly + statistics.
* Generated for debug contexts only.
unsigned vgt_gsvs_ring_offset_1;
unsigned vgt_gsvs_ring_offset_2;
unsigned vgt_gsvs_ring_offset_3;
- unsigned vgt_gs_out_prim_type;
unsigned vgt_gsvs_ring_itemsize;
unsigned vgt_gs_max_vert_out;
unsigned vgt_gs_vert_itemsize;
unsigned vgt_esgs_ring_itemsize;
} gs;
+ struct {
+ unsigned ge_max_output_per_subgroup;
+ unsigned ge_ngg_subgrp_cntl;
+ unsigned vgt_primitiveid_en;
+ unsigned vgt_gs_onchip_cntl;
+ unsigned vgt_gs_instance_cnt;
+ unsigned vgt_esgs_ring_itemsize;
+ unsigned spi_vs_out_config;
+ unsigned spi_shader_idx_format;
+ unsigned spi_shader_pos_format;
+ unsigned pa_cl_vte_cntl;
+ unsigned pa_cl_ngg_cntl;
+ unsigned vgt_gs_max_vert_out; /* for API GS */
+ } ngg;
+
struct {
unsigned vgt_gs_mode;
unsigned vgt_primitiveid_en;
/*For save precompute registers value */
unsigned vgt_tf_param; /* VGT_TF_PARAM */
unsigned vgt_vertex_reuse_block_cntl; /* VGT_VERTEX_REUSE_BLOCK_CNTL */
+ unsigned pa_cl_vs_out_cntl;
+ unsigned ge_cntl;
};
struct si_shader_part {
struct ac_llvm_compiler *compiler,
struct si_shader_selector *gs_selector,
struct pipe_debug_callback *debug);
-int si_compile_tgsi_shader(struct si_screen *sscreen,
- struct ac_llvm_compiler *compiler,
- struct si_shader *shader,
- struct pipe_debug_callback *debug);
+int si_compile_shader(struct si_screen *sscreen,
+ struct ac_llvm_compiler *compiler,
+ struct si_shader *shader,
+ struct pipe_debug_callback *debug);
bool si_shader_create(struct si_screen *sscreen, struct ac_llvm_compiler *compiler,
struct si_shader *shader,
struct pipe_debug_callback *debug);
bool si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader,
uint64_t scratch_va);
void si_shader_dump(struct si_screen *sscreen, struct si_shader *shader,
- struct pipe_debug_callback *debug, unsigned processor,
+ struct pipe_debug_callback *debug,
FILE *f, bool check_debug_option);
void si_shader_dump_stats_for_shader_db(struct si_screen *screen,
struct si_shader *shader,
struct pipe_debug_callback *debug);
void si_multiwave_lds_size_workaround(struct si_screen *sscreen,
unsigned *lds_size);
-const char *si_get_shader_name(const struct si_shader *shader, unsigned processor);
+const char *si_get_shader_name(const struct si_shader *shader);
void si_shader_binary_clean(struct si_shader_binary *binary);
/* si_shader_nir.c */
void si_nir_scan_shader(const struct nir_shader *nir,
- struct tgsi_shader_info *info);
-void si_nir_scan_tess_ctrl(const struct nir_shader *nir,
- struct tgsi_tessctrl_info *out);
-void si_lower_nir(struct si_shader_selector *sel);
-void si_nir_opts(struct nir_shader *nir);
+ struct si_shader_info *info);
+void si_nir_adjust_driver_locations(struct nir_shader *nir);
+void si_finalize_nir(struct pipe_screen *screen, void *nirptr, bool optimize);
/* si_state_shaders.c */
void gfx9_get_gs_info(struct si_shader_selector *es,
{
if (key->as_ls)
return &sel->main_shader_part_ls;
+ if (key->as_es && key->as_ngg)
+ return &sel->main_shader_part_ngg_es;
if (key->as_es)
return &sel->main_shader_part_es;
if (key->as_ngg)
return &sel->main_shader_part;
}
+static inline bool
+gfx10_is_ngg_passthrough(struct si_shader *shader)
+{
+ struct si_shader_selector *sel = shader->selector;
+
+ return sel->type != PIPE_SHADER_GEOMETRY &&
+ !sel->so.num_outputs &&
+ !sel->info.writes_edgeflag &&
+ (sel->type != PIPE_SHADER_VERTEX ||
+ !shader->key.mono.u.vs_export_prim_id);
+}
+
static inline bool
si_shader_uses_bindless_samplers(struct si_shader_selector *selector)
{