#include "common/gen_clflush.h"
#include "common/gen_decoder.h"
#include "common/gen_gem.h"
+#include "common/gen_l3_config.h"
#include "dev/gen_device_info.h"
#include "blorp/blorp.h"
#include "compiler/brw_compiler.h"
struct anv_instance;
struct gen_aux_map_context;
-struct gen_l3_config;
struct gen_perf_config;
#include <vulkan/vulkan.h>
#define MAX_PUSH_DESCRIPTORS 32 /* Minimum requirement */
#define MAX_INLINE_UNIFORM_BLOCK_SIZE 4096
#define MAX_INLINE_UNIFORM_BLOCK_DESCRIPTORS 32
+#define ANV_UBO_BOUNDS_CHECK_ALIGNMENT 32
+#define ANV_SSBO_BOUNDS_CHECK_ALIGNMENT 4
/* From the Skylake PRM Vol. 7 "Binding Table Surface State Model":
*
return v - (v % a);
}
+static inline uint32_t
+align_down_u32(uint32_t v, uint32_t a)
+{
+ assert(a != 0 && a == (a & -a));
+ return v & ~(a - 1);
+}
+
static inline uint32_t
align_u32(uint32_t v, uint32_t a)
{
assert(a != 0 && a == (a & -a));
- return (v + a - 1) & ~(a - 1);
+ return align_down_u32(v + a - 1, a);
}
static inline uint64_t
*/
uint64_t offset;
+ /** Size of the buffer not including implicit aux */
uint64_t size;
/* Map for internally mapped BOs.
*/
void *map;
+ /** Size of the implicit CCS range at the end of the buffer
+ *
+ * On Gen12, CCS data is always a direct 1/256 scale-down. A single 64K
+ * page of main surface data maps to a 256B chunk of CCS data and that
+ * mapping is provided on TGL-LP by the AUX table which maps virtual memory
+ * addresses in the main surface to virtual memory addresses for CCS data.
+ *
+ * Because we can't change these maps around easily and because Vulkan
+ * allows two VkImages to be bound to overlapping memory regions (as long
+ * as the app is careful), it's not feasible to make this mapping part of
+ * the image. (On Gen11 and earlier, the mapping was provided via
+ * RENDER_SURFACE_STATE so each image had its own main -> CCS mapping.)
+ * Instead, we attach the CCS data directly to the buffer object and setup
+ * the AUX table mapping at BO creation time.
+ *
+ * This field is for internal tracking use by the BO allocator only and
+ * should not be touched by other parts of the code. If something wants to
+ * know if a BO has implicit CCS data, it should instead look at the
+ * has_implicit_ccs boolean below.
+ *
+ * This data is not included in maps of this buffer.
+ */
+ uint32_t _ccs_size;
+
/** Flags to pass to the kernel through drm_i915_exec_object2::flags */
uint32_t flags;
/** See also ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS */
bool has_client_visible_address:1;
+
+ /** True if this BO has implicit CCS data attached to it */
+ bool has_implicit_ccs:1;
};
static inline struct anv_bo *
struct anv_fixed_size_state_pool buckets[ANV_STATE_BUCKETS];
};
-struct anv_state_stream_block;
-
struct anv_state_stream {
struct anv_state_pool *state_pool;
uint32_t next;
/* List of all blocks allocated from this pool */
- struct anv_state_stream_block *block_list;
+ struct util_dynarray all_blocks;
};
/* The block_pool functions exported for testing only. The block pool should
uint32_t block_size, uint32_t *padding);
int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool,
uint32_t block_size);
-void* anv_block_pool_map(struct anv_block_pool *pool, int32_t offset);
+void* anv_block_pool_map(struct anv_block_pool *pool, int32_t offset, uint32_t
+size);
VkResult anv_state_pool_init(struct anv_state_pool *pool,
struct anv_device *device,
/** True if we can use bindless access for samplers */
bool has_bindless_samplers;
+ /** True if this device has implicit AUX
+ *
+ * If true, CCS is handled as an implicit attachment to the BO rather than
+ * as an explicitly bound surface.
+ */
+ bool has_implicit_ccs;
+
bool always_flush_cache;
struct anv_device_extension_table supported_extensions;
const void *key, uint32_t key_size);
struct anv_shader_bin *
anv_pipeline_cache_upload_kernel(struct anv_pipeline_cache *cache,
+ gl_shader_stage stage,
const void *key_data, uint32_t key_size,
const void *kernel_data, uint32_t kernel_size,
const void *constant_data,
struct anv_shader_bin *
anv_device_upload_kernel(struct anv_device *device,
struct anv_pipeline_cache *cache,
+ gl_shader_stage stage,
const void *key_data, uint32_t key_size,
const void *kernel_data, uint32_t kernel_size,
const void *constant_data,
/** Has an address which is visible to the client */
ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS = (1 << 8),
+
+ /** This buffer has implicit CCS data attached to it */
+ ANV_BO_ALLOC_IMPLICIT_CCS = (1 << 9),
};
VkResult anv_device_alloc_bo(struct anv_device *device, uint64_t size,
uint32_t *handles, uint32_t num_handles,
int64_t abs_timeout_ns, bool wait_all);
-bool anv_vma_alloc(struct anv_device *device, struct anv_bo *bo,
- uint64_t client_address);
-void anv_vma_free(struct anv_device *device, struct anv_bo *bo);
+uint64_t anv_vma_alloc(struct anv_device *device,
+ uint64_t size, uint64_t align,
+ enum anv_bo_alloc_flags alloc_flags,
+ uint64_t client_address);
+void anv_vma_free(struct anv_device *device,
+ uint64_t address, uint64_t size);
struct anv_reloc_list {
uint32_t num_relocs;
ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT = (1 << 12),
ANV_PIPE_DEPTH_STALL_BIT = (1 << 13),
ANV_PIPE_CS_STALL_BIT = (1 << 20),
+ ANV_PIPE_END_OF_PIPE_SYNC_BIT = (1 << 21),
/* This bit does not exist directly in PIPE_CONTROL. Instead it means that
* a flush has happened but not a CS stall. The next time we do any sort
* of invalidation we need to insert a CS stall at that time. Otherwise,
* we would have to CS stall on every flush which could be bad.
*/
- ANV_PIPE_NEEDS_CS_STALL_BIT = (1 << 21),
+ ANV_PIPE_NEEDS_END_OF_PIPE_SYNC_BIT = (1 << 22),
/* This bit does not exist directly in PIPE_CONTROL. It means that render
* target operations related to transfer commands with VkBuffer as
* streamer might need to be aware of this to trigger the appropriate stall
* before they can proceed with the copy.
*/
- ANV_PIPE_RENDER_TARGET_BUFFER_WRITES = (1 << 22),
+ ANV_PIPE_RENDER_TARGET_BUFFER_WRITES = (1 << 23),
+
+ /* This bit does not exist directly in PIPE_CONTROL. It means that Gen12
+ * AUX-TT data has changed and we need to invalidate AUX-TT data. This is
+ * done by writing the AUX-TT register.
+ */
+ ANV_PIPE_AUX_TABLE_INVALIDATE_BIT = (1 << 24),
+
+ /* This bit does not exist directly in PIPE_CONTROL. It means that a
+ * PIPE_CONTROL with a post-sync operation will follow. This is used to
+ * implement a workaround for Gen9.
+ */
+ ANV_PIPE_POST_SYNC_BIT = (1 << 25),
};
#define ANV_PIPE_FLUSH_BITS ( \
ANV_PIPE_VF_CACHE_INVALIDATE_BIT | \
ANV_PIPE_DATA_CACHE_FLUSH_BIT | \
ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT | \
- ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT)
+ ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT | \
+ ANV_PIPE_AUX_TABLE_INVALIDATE_BIT)
static inline enum anv_pipe_bits
anv_pipe_flush_bits_for_access_flags(VkAccessFlags flags)
/** Dynamic offsets for dynamic UBOs and SSBOs */
uint32_t dynamic_offsets[MAX_DYNAMIC_BUFFERS];
+ /** Pad out to a multiple of 32 bytes */
+ uint32_t push_ubo_sizes[4];
+
struct {
/** Base workgroup ID
*
* uploading the push constants for compute shaders.
*/
uint32_t subgroup_id;
-
- /** Pad out to a multiple of 32 bytes */
- uint32_t pad[4];
} cs;
};
* per-stage array in anv_cmd_state.
*/
struct anv_cmd_pipeline_state {
- struct anv_pipeline *pipeline;
-
struct anv_descriptor_set *descriptors[MAX_SETS];
struct anv_push_descriptor_set *push_descriptors[MAX_SETS];
};
struct anv_cmd_graphics_state {
struct anv_cmd_pipeline_state base;
+ struct anv_graphics_pipeline *pipeline;
+
anv_cmd_dirty_mask_t dirty;
uint32_t vb_dirty;
struct anv_cmd_compute_state {
struct anv_cmd_pipeline_state base;
+ struct anv_compute_pipeline *pipeline;
+
bool pipeline_dirty;
struct anv_address num_workgroups;
struct anv_shader_bin {
uint32_t ref_cnt;
+ gl_shader_stage stage;
+
const struct anv_shader_bin_key *key;
struct anv_state kernel;
struct anv_shader_bin *
anv_shader_bin_create(struct anv_device *device,
+ gl_shader_stage stage,
const void *key, uint32_t key_size,
const void *kernel, uint32_t kernel_size,
const void *constant_data, uint32_t constant_data_size,
const struct brw_stage_prog_data *prog_data,
- uint32_t prog_data_size, const void *prog_data_param,
+ uint32_t prog_data_size,
const struct brw_compile_stats *stats, uint32_t num_stats,
const struct nir_xfb_info *xfb_info,
const struct anv_pipeline_bind_map *bind_map);
anv_shader_bin_destroy(device, shader);
}
-/* 5 possible simultaneous shader stages and FS may have up to 3 binaries */
-#define MAX_PIPELINE_EXECUTABLES 7
-
struct anv_pipeline_executable {
gl_shader_stage stage;
char *disasm;
};
+enum anv_pipeline_type {
+ ANV_PIPELINE_GRAPHICS,
+ ANV_PIPELINE_COMPUTE,
+};
+
struct anv_pipeline {
struct anv_device * device;
+
struct anv_batch batch;
- uint32_t batch_data[512];
struct anv_reloc_list batch_relocs;
- anv_cmd_dirty_mask_t dynamic_state_mask;
- struct anv_dynamic_state dynamic_state;
void * mem_ctx;
+ enum anv_pipeline_type type;
VkPipelineCreateFlags flags;
- struct anv_subpass * subpass;
- bool needs_data_cache;
+ struct util_dynarray executables;
- struct anv_shader_bin * shaders[MESA_SHADER_STAGES];
+ const struct gen_l3_config * l3_config;
+};
- uint32_t num_executables;
- struct anv_pipeline_executable executables[MAX_PIPELINE_EXECUTABLES];
+struct anv_graphics_pipeline {
+ struct anv_pipeline base;
- struct {
- const struct gen_l3_config * l3_config;
- uint32_t total_size;
- } urb;
+ uint32_t batch_data[512];
- VkShaderStageFlags active_stages;
- struct anv_state blend_state;
+ anv_cmd_dirty_mask_t dynamic_state_mask;
+ struct anv_dynamic_state dynamic_state;
- uint32_t vb_used;
- struct anv_pipeline_vertex_binding {
- uint32_t stride;
- bool instanced;
- uint32_t instance_divisor;
- } vb[MAX_VBS];
+ uint32_t topology;
- uint8_t xfb_used;
+ struct anv_subpass * subpass;
- bool primitive_restart;
- uint32_t topology;
+ struct anv_shader_bin * shaders[MESA_SHADER_STAGES];
- uint32_t cs_right_mask;
+ VkShaderStageFlags active_stages;
+ bool primitive_restart;
bool writes_depth;
bool depth_test_enable;
bool writes_stencil;
bool kill_pixel;
bool depth_bounds_test_enable;
+ struct anv_state blend_state;
+
+ uint32_t vb_used;
+ struct anv_pipeline_vertex_binding {
+ uint32_t stride;
+ bool instanced;
+ uint32_t instance_divisor;
+ } vb[MAX_VBS];
+
struct {
uint32_t sf[7];
uint32_t depth_stencil_state[3];
struct {
uint32_t wm_depth_stencil[4];
} gen9;
+};
+
+struct anv_compute_pipeline {
+ struct anv_pipeline base;
+ struct anv_shader_bin * cs;
+ uint32_t cs_right_mask;
+ uint32_t batch_data[9];
uint32_t interface_descriptor_data[8];
};
+#define ANV_DECL_PIPELINE_DOWNCAST(pipe_type, pipe_enum) \
+ static inline struct anv_##pipe_type##_pipeline * \
+ anv_pipeline_to_##pipe_type(struct anv_pipeline *pipeline) \
+ { \
+ assert(pipeline->type == pipe_enum); \
+ return (struct anv_##pipe_type##_pipeline *) pipeline; \
+ }
+
+ANV_DECL_PIPELINE_DOWNCAST(graphics, ANV_PIPELINE_GRAPHICS)
+ANV_DECL_PIPELINE_DOWNCAST(compute, ANV_PIPELINE_COMPUTE)
+
static inline bool
-anv_pipeline_has_stage(const struct anv_pipeline *pipeline,
+anv_pipeline_has_stage(const struct anv_graphics_pipeline *pipeline,
gl_shader_stage stage)
{
return (pipeline->active_stages & mesa_to_vk_shader_stage(stage)) != 0;
}
-#define ANV_DECL_GET_PROG_DATA_FUNC(prefix, stage) \
-static inline const struct brw_##prefix##_prog_data * \
-get_##prefix##_prog_data(const struct anv_pipeline *pipeline) \
-{ \
- if (anv_pipeline_has_stage(pipeline, stage)) { \
- return (const struct brw_##prefix##_prog_data *) \
- pipeline->shaders[stage]->prog_data; \
- } else { \
- return NULL; \
- } \
+#define ANV_DECL_GET_GRAPHICS_PROG_DATA_FUNC(prefix, stage) \
+static inline const struct brw_##prefix##_prog_data * \
+get_##prefix##_prog_data(const struct anv_graphics_pipeline *pipeline) \
+{ \
+ if (anv_pipeline_has_stage(pipeline, stage)) { \
+ return (const struct brw_##prefix##_prog_data *) \
+ pipeline->shaders[stage]->prog_data; \
+ } else { \
+ return NULL; \
+ } \
}
-ANV_DECL_GET_PROG_DATA_FUNC(vs, MESA_SHADER_VERTEX)
-ANV_DECL_GET_PROG_DATA_FUNC(tcs, MESA_SHADER_TESS_CTRL)
-ANV_DECL_GET_PROG_DATA_FUNC(tes, MESA_SHADER_TESS_EVAL)
-ANV_DECL_GET_PROG_DATA_FUNC(gs, MESA_SHADER_GEOMETRY)
-ANV_DECL_GET_PROG_DATA_FUNC(wm, MESA_SHADER_FRAGMENT)
-ANV_DECL_GET_PROG_DATA_FUNC(cs, MESA_SHADER_COMPUTE)
+ANV_DECL_GET_GRAPHICS_PROG_DATA_FUNC(vs, MESA_SHADER_VERTEX)
+ANV_DECL_GET_GRAPHICS_PROG_DATA_FUNC(tcs, MESA_SHADER_TESS_CTRL)
+ANV_DECL_GET_GRAPHICS_PROG_DATA_FUNC(tes, MESA_SHADER_TESS_EVAL)
+ANV_DECL_GET_GRAPHICS_PROG_DATA_FUNC(gs, MESA_SHADER_GEOMETRY)
+ANV_DECL_GET_GRAPHICS_PROG_DATA_FUNC(wm, MESA_SHADER_FRAGMENT)
+
+static inline const struct brw_cs_prog_data *
+get_cs_prog_data(const struct anv_compute_pipeline *pipeline)
+{
+ assert(pipeline->cs);
+ return (const struct brw_cs_prog_data *) pipeline->cs->prog_data;
+}
static inline const struct brw_vue_prog_data *
-anv_pipeline_get_last_vue_prog_data(const struct anv_pipeline *pipeline)
+anv_pipeline_get_last_vue_prog_data(const struct anv_graphics_pipeline *pipeline)
{
if (anv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY))
return &get_gs_prog_data(pipeline)->base;
}
VkResult
-anv_pipeline_init(struct anv_pipeline *pipeline, struct anv_device *device,
+anv_pipeline_init(struct anv_graphics_pipeline *pipeline, struct anv_device *device,
struct anv_pipeline_cache *cache,
const VkGraphicsPipelineCreateInfo *pCreateInfo,
const VkAllocationCallbacks *alloc);
VkResult
-anv_pipeline_compile_cs(struct anv_pipeline *pipeline,
+anv_pipeline_compile_cs(struct anv_compute_pipeline *pipeline,
struct anv_pipeline_cache *cache,
const VkComputePipelineCreateInfo *info,
const struct anv_shader_module *module,
bool can_ycbcr;
};
+/**
+ * Return the aspect's _format_ plane, not its _memory_ plane (using the
+ * vocabulary of VK_EXT_image_drm_format_modifier). As a consequence, \a
+ * aspect_mask may contain VK_IMAGE_ASPECT_PLANE_*, but must not contain
+ * VK_IMAGE_ASPECT_MEMORY_PLANE_* .
+ */
static inline uint32_t
anv_image_aspect_to_plane(VkImageAspectFlags image_aspects,
VkImageAspectFlags aspect_mask)
*/
bool disjoint;
- /* All the formats that can be used when creating views of this image
- * are CCS_E compatible.
- */
- bool ccs_e_compatible;
-
/* Image was created with external format. */
bool external_format;
*/
struct anv_address address;
- /**
- * Address of the main surface used to fill the aux map table. This is
- * used at destruction of the image since the Vulkan spec does not
- * guarantee that the address.bo field we still be valid at destruction.
- */
- uint64_t aux_map_surface_address;
-
/**
* When destroying the image, also free the bo.
* */
if (!(image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT))
return false;
+ /* For Gen8-11, there are some restrictions around sampling from HiZ.
+ * The Skylake PRM docs for RENDER_SURFACE_STATE::AuxiliarySurfaceMode
+ * say:
+ *
+ * "If this field is set to AUX_HIZ, Number of Multisamples must
+ * be MULTISAMPLECOUNT_1, and Surface Type cannot be SURFTYPE_3D."
+ */
+ if (image->type == VK_IMAGE_TYPE_3D)
+ return false;
+
/* Allow this feature on BDW even though it is disabled in the BDW devinfo
* struct. There's documentation which suggests that this feature actually
* reduces performance on BDW, but it has only been observed to help so
void
anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
const struct anv_image *image,
- enum isl_format format,
+ enum isl_format format, struct isl_swizzle swizzle,
VkImageAspectFlagBits aspect,
uint32_t base_layer, uint32_t layer_count,
enum isl_aux_op mcs_op, union isl_color_value *clear_value,
void
anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
const struct anv_image *image,
- enum isl_format format,
+ enum isl_format format, struct isl_swizzle swizzle,
VkImageAspectFlagBits aspect, uint32_t level,
uint32_t base_layer, uint32_t layer_count,
enum isl_aux_op ccs_op, union isl_color_value *clear_value,
const VkAllocationCallbacks* alloc,
VkImage *pImage);
-const struct anv_surface *
-anv_image_get_surface_for_aspect_mask(const struct anv_image *image,
- VkImageAspectFlags aspect_mask);
-
enum isl_format
anv_isl_format_for_descriptor_type(VkDescriptorType type);
projects / mesa.git / blobdiff