#include <pthread.h>
#include <assert.h>
#include <stdint.h>
-#include <i915_drm.h>
+#include "drm-uapi/i915_drm.h"
#ifdef HAVE_VALGRIND
#include <valgrind.h>
#endif
#include "common/gen_clflush.h"
+#include "common/gen_decoder.h"
#include "common/gen_gem.h"
#include "dev/gen_device_info.h"
#include "blorp/blorp.h"
#define ANV_HZ_FC_VAL 1.0f
#define MAX_VBS 28
+#define MAX_XFB_BUFFERS 4
+#define MAX_XFB_STREAMS 4
#define MAX_SETS 8
#define MAX_RTS 8
#define MAX_VIEWPORTS 16
#define MAX_IMAGES 64
#define MAX_GEN8_IMAGES 8
#define MAX_PUSH_DESCRIPTORS 32 /* Minimum requirement */
+#define MAX_INLINE_UNIFORM_BLOCK_SIZE 4096
+#define MAX_INLINE_UNIFORM_BLOCK_DESCRIPTORS 32
/* The kernel relocation API has a limitation of a 32-bit delta value
* applied to the address before it is written which, in spite of it being
#define ANV_SVGS_VB_INDEX MAX_VBS
#define ANV_DRAWID_VB_INDEX (MAX_VBS + 1)
+/* We reserve this MI ALU register for the purpose of handling predication.
+ * Other code which uses the MI ALU should leave it alone.
+ */
+#define ANV_PREDICATE_RESULT_REG MI_ALU_REG15
+
#define anv_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
static inline uint32_t
*/
uint32_t center_bo_offset;
+ /* Current memory map of the block pool. This pointer may or may not
+ * point to the actual beginning of the block pool memory. If
+ * anv_block_pool_alloc_back has ever been called, then this pointer
+ * will point to the "center" position of the buffer and all offsets
+ * (negative or positive) given out by the block pool alloc functions
+ * will be valid relative to this pointer.
+ *
+ * In particular, map == bo.map + center_offset
+ *
+ * DO NOT access this pointer directly. Use anv_block_pool_map() instead,
+ * since it will handle the softpin case as well, where this points to NULL.
+ */
+ void *map;
int fd;
/**
uint64_t bo_flags);
void anv_block_pool_finish(struct anv_block_pool *pool);
int32_t anv_block_pool_alloc(struct anv_block_pool *pool,
- uint32_t block_size);
+ uint32_t block_size, uint32_t *padding);
int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool,
uint32_t block_size);
void* anv_block_pool_map(struct anv_block_pool *pool, int32_t offset);
struct anv_bo_cache *cache,
uint64_t size, uint64_t bo_flags,
struct anv_bo **bo);
+VkResult anv_bo_cache_import_host_ptr(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ void *host_ptr, uint32_t size,
+ uint64_t bo_flags, struct anv_bo **bo_out);
VkResult anv_bo_cache_import(struct anv_device *device,
struct anv_bo_cache *cache,
int fd, uint64_t bo_flags,
struct hash_table * cache;
};
+struct nir_xfb_info;
struct anv_pipeline_bind_map;
void anv_pipeline_cache_init(struct anv_pipeline_cache *cache,
uint32_t constant_data_size,
const struct brw_stage_prog_data *prog_data,
uint32_t prog_data_size,
+ const struct nir_xfb_info *xfb_info,
const struct anv_pipeline_bind_map *bind_map);
struct anv_shader_bin *
uint32_t constant_data_size,
const struct brw_stage_prog_data *prog_data,
uint32_t prog_data_size,
+ const struct nir_xfb_info *xfb_info,
const struct anv_pipeline_bind_map *bind_map);
struct nir_shader;
struct anv_bo trivial_batch_bo;
struct anv_bo hiz_clear_bo;
+ /* Set of pointers to anv_buffer objects for all pinned buffers. Pinned
+ * buffers are always resident because they could be used at any time via
+ * VK_EXT_buffer_device_address.
+ */
+ struct set * pinned_buffers;
+
struct anv_pipeline_cache default_pipeline_cache;
struct blorp_context blorp;
pthread_mutex_t mutex;
pthread_cond_t queue_submit;
bool _lost;
+
+ struct gen_batch_decode_ctx decoder_ctx;
+ /*
+ * When decoding a anv_cmd_buffer, we might need to search for BOs through
+ * the cmd_buffer's list.
+ */
+ struct anv_cmd_buffer *cmd_buffer_being_decoded;
};
static inline struct anv_state_pool *
return device->default_mocs;
}
-static void inline
-anv_state_flush(struct anv_device *device, struct anv_state state)
-{
- if (device->info.has_llc)
- return;
-
- gen_flush_range(state.map, state.alloc_size);
-}
-
void anv_device_init_blorp(struct anv_device *device);
void anv_device_finish_blorp(struct anv_device *device);
*/
/* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */
-#define GEN9_MOCS 2
+#define GEN9_MOCS (2 << 1)
/* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */
-#define GEN9_EXTERNAL_MOCS 1
+#define GEN9_EXTERNAL_MOCS (1 << 1)
/* Cannonlake MOCS defines are duplicates of Skylake MOCS defines. */
#define GEN10_MOCS GEN9_MOCS
* which we must release when memory is freed.
*/
struct AHardwareBuffer * ahw;
+
+ /* If set, this memory comes from a host pointer. */
+ void * host_ptr;
};
/**
float PointWidth;
};
+enum anv_descriptor_data {
+ /** The descriptor contains a BTI reference to a surface state */
+ ANV_DESCRIPTOR_SURFACE_STATE = (1 << 0),
+ /** The descriptor contains a BTI reference to a sampler state */
+ ANV_DESCRIPTOR_SAMPLER_STATE = (1 << 1),
+ /** The descriptor contains an actual buffer view */
+ ANV_DESCRIPTOR_BUFFER_VIEW = (1 << 2),
+ /** The descriptor contains auxiliary image layout data */
+ ANV_DESCRIPTOR_IMAGE_PARAM = (1 << 3),
+ /** The descriptor contains auxiliary image layout data */
+ ANV_DESCRIPTOR_INLINE_UNIFORM = (1 << 4),
+};
+
struct anv_descriptor_set_binding_layout {
#ifndef NDEBUG
/* The type of the descriptors in this binding */
VkDescriptorType type;
#endif
- /* Number of array elements in this binding */
+ /* Bitfield representing the type of data this descriptor contains */
+ enum anv_descriptor_data data;
+
+ /* Number of array elements in this binding (or size in bytes for inline
+ * uniform data)
+ */
uint16_t array_size;
/* Index into the flattend descriptor set */
int16_t dynamic_offset_index;
/* Index into the descriptor set buffer views */
- int16_t buffer_index;
-
- struct {
- /* Index into the binding table for the associated surface */
- int16_t surface_index;
+ int16_t buffer_view_index;
- /* Index into the sampler table for the associated sampler */
- int16_t sampler_index;
-
- /* Index into the image table for the associated image */
- int16_t image_index;
- } stage[MESA_SHADER_STAGES];
+ /* Offset into the descriptor buffer where this descriptor lives */
+ uint32_t descriptor_offset;
/* Immutable samplers (or NULL if no immutable samplers) */
struct anv_sampler **immutable_samplers;
};
+unsigned anv_descriptor_size(const struct anv_descriptor_set_binding_layout *layout);
+
+unsigned anv_descriptor_type_size(const struct anv_physical_device *pdevice,
+ VkDescriptorType type);
+
struct anv_descriptor_set_layout {
/* Descriptor set layouts can be destroyed at almost any time */
uint32_t ref_cnt;
/* Shader stages affected by this descriptor set */
uint16_t shader_stages;
- /* Number of buffers in this descriptor set */
- uint16_t buffer_count;
+ /* Number of buffer views in this descriptor set */
+ uint16_t buffer_view_count;
/* Number of dynamic offsets used by this descriptor set */
uint16_t dynamic_offset_count;
+ /* Size of the descriptor buffer for this descriptor set */
+ uint32_t descriptor_buffer_size;
+
/* Bindings in this descriptor set */
struct anv_descriptor_set_binding_layout binding[0];
};
};
struct anv_descriptor_set {
+ struct anv_descriptor_pool *pool;
struct anv_descriptor_set_layout *layout;
uint32_t size;
- uint32_t buffer_count;
+
+ /* State relative to anv_descriptor_pool::bo */
+ struct anv_state desc_mem;
+ /* Surface state for the descriptor buffer */
+ struct anv_state desc_surface_state;
+
+ uint32_t buffer_view_count;
struct anv_buffer_view *buffer_views;
struct anv_descriptor descriptors[0];
};
/* Put this field right behind anv_descriptor_set so it fills up the
* descriptors[0] field. */
struct anv_descriptor descriptors[MAX_PUSH_DESCRIPTORS];
+
+ /** True if the descriptor set buffer has been referenced by a draw or
+ * dispatch command.
+ */
+ bool set_used_on_gpu;
+
struct anv_buffer_view buffer_views[MAX_PUSH_DESCRIPTORS];
};
uint32_t next;
uint32_t free_list;
+ struct anv_bo bo;
+ struct util_vma_heap bo_heap;
+
struct anv_state_stream surface_state_stream;
void *surface_state_free_list;
anv_descriptor_set_layout_size(const struct anv_descriptor_set_layout *layout);
void
-anv_descriptor_set_write_image_view(struct anv_descriptor_set *set,
- const struct gen_device_info * const devinfo,
+anv_descriptor_set_write_image_view(struct anv_device *device,
+ struct anv_descriptor_set *set,
const VkDescriptorImageInfo * const info,
VkDescriptorType type,
uint32_t binding,
uint32_t element);
void
-anv_descriptor_set_write_buffer_view(struct anv_descriptor_set *set,
+anv_descriptor_set_write_buffer_view(struct anv_device *device,
+ struct anv_descriptor_set *set,
VkDescriptorType type,
struct anv_buffer_view *buffer_view,
uint32_t binding,
uint32_t element);
void
-anv_descriptor_set_write_buffer(struct anv_descriptor_set *set,
- struct anv_device *device,
+anv_descriptor_set_write_buffer(struct anv_device *device,
+ struct anv_descriptor_set *set,
struct anv_state_stream *alloc_stream,
VkDescriptorType type,
struct anv_buffer *buffer,
uint32_t element,
VkDeviceSize offset,
VkDeviceSize range);
+void
+anv_descriptor_set_write_inline_uniform_data(struct anv_device *device,
+ struct anv_descriptor_set *set,
+ uint32_t binding,
+ const void *data,
+ size_t offset,
+ size_t size);
void
-anv_descriptor_set_write_template(struct anv_descriptor_set *set,
- struct anv_device *device,
+anv_descriptor_set_write_template(struct anv_device *device,
+ struct anv_descriptor_set *set,
struct anv_state_stream *alloc_stream,
const struct anv_descriptor_update_template *template,
const void *data);
struct anv_descriptor_pool *pool,
struct anv_descriptor_set *set);
+#define ANV_DESCRIPTOR_SET_DESCRIPTORS (UINT8_MAX - 3)
+#define ANV_DESCRIPTOR_SET_NUM_WORK_GROUPS (UINT8_MAX - 2)
#define ANV_DESCRIPTOR_SET_SHADER_CONSTANTS (UINT8_MAX - 1)
#define ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS UINT8_MAX
uint32_t num_sets;
- struct {
- bool has_dynamic_offsets;
- } stage[MESA_SHADER_STAGES];
-
unsigned char sha1[20];
};
ANV_CMD_DIRTY_PIPELINE = 1 << 9,
ANV_CMD_DIRTY_INDEX_BUFFER = 1 << 10,
ANV_CMD_DIRTY_RENDER_TARGETS = 1 << 11,
+ ANV_CMD_DIRTY_XFB_ENABLE = 1 << 12,
};
typedef uint32_t anv_cmd_dirty_mask_t;
ANV_PIPE_NEEDS_CS_STALL_BIT = (1 << 21),
/* This bit does not exist directly in PIPE_CONTROL. It means that render
- * target operations are ongoing. Some operations like copies on the
- * command streamer might need to be aware of this to trigger the
- * appropriate stall before they can proceed with the copy.
+ * target operations related to transfer commands with VkBuffer as
+ * destination are ongoing. Some operations like copies on the command
+ * streamer might need to be aware of this to trigger the appropriate stall
+ * before they can proceed with the copy.
*/
- ANV_PIPE_RENDER_TARGET_WRITES = (1 << 22),
+ ANV_PIPE_RENDER_TARGET_BUFFER_WRITES = (1 << 22),
};
#define ANV_PIPE_FLUSH_BITS ( \
for_each_bit(b, flags) {
switch ((VkAccessFlagBits)(1 << b)) {
case VK_ACCESS_SHADER_WRITE_BIT:
+ /* We're transitioning a buffer that was previously used as write
+ * destination through the data port. To make its content available
+ * to future operations, flush the data cache.
+ */
pipe_bits |= ANV_PIPE_DATA_CACHE_FLUSH_BIT;
break;
case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT:
+ /* We're transitioning a buffer that was previously used as render
+ * target. To make its content available to future operations, flush
+ * the render target cache.
+ */
pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
break;
case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT:
+ /* We're transitioning a buffer that was previously used as depth
+ * buffer. To make its content available to future operations, flush
+ * the depth cache.
+ */
pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
break;
case VK_ACCESS_TRANSFER_WRITE_BIT:
+ /* We're transitioning a buffer that was previously used as a
+ * transfer write destination. Generic write operations include color
+ * & depth operations as well as buffer operations like :
+ * - vkCmdClearColorImage()
+ * - vkCmdClearDepthStencilImage()
+ * - vkCmdBlitImage()
+ * - vkCmdCopy*(), vkCmdUpdate*(), vkCmdFill*()
+ *
+ * Most of these operations are implemented using Blorp which writes
+ * through the render target, so flush that cache to make it visible
+ * to future operations. And for depth related operations we also
+ * need to flush the depth cache.
+ */
pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
break;
case VK_ACCESS_MEMORY_WRITE_BIT:
+ /* We're transitioning a buffer for generic write operations. Flush
+ * all the caches.
+ */
pipe_bits |= ANV_PIPE_FLUSH_BITS;
break;
default:
for_each_bit(b, flags) {
switch ((VkAccessFlagBits)(1 << b)) {
case VK_ACCESS_INDIRECT_COMMAND_READ_BIT:
+ /* Indirect draw commands take a buffer as input that we're going to
+ * read from the command streamer to load some of the HW registers
+ * (see genX_cmd_buffer.c:load_indirect_parameters). This requires a
+ * command streamer stall so that all the cache flushes have
+ * completed before the command streamer loads from memory.
+ */
+ pipe_bits |= ANV_PIPE_CS_STALL_BIT;
+ /* Indirect draw commands also set gl_BaseVertex & gl_BaseIndex
+ * through a vertex buffer, so invalidate that cache.
+ */
+ pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
+ /* For CmdDipatchIndirect, we also load gl_NumWorkGroups through a
+ * UBO from the buffer, so we need to invalidate constant cache.
+ */
+ pipe_bits |= ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
+ break;
case VK_ACCESS_INDEX_READ_BIT:
case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT:
+ /* We transitioning a buffer to be used for as input for vkCmdDraw*
+ * commands, so we invalidate the VF cache to make sure there is no
+ * stale data when we start rendering.
+ */
pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
break;
case VK_ACCESS_UNIFORM_READ_BIT:
+ /* We transitioning a buffer to be used as uniform data. Because
+ * uniform is accessed through the data port & sampler, we need to
+ * invalidate the texture cache (sampler) & constant cache (data
+ * port) to avoid stale data.
+ */
pipe_bits |= ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
break;
case VK_ACCESS_SHADER_READ_BIT:
case VK_ACCESS_INPUT_ATTACHMENT_READ_BIT:
case VK_ACCESS_TRANSFER_READ_BIT:
+ /* Transitioning a buffer to be read through the sampler, so
+ * invalidate the texture cache, we don't want any stale data.
+ */
pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
break;
case VK_ACCESS_MEMORY_READ_BIT:
+ /* Transitioning a buffer for generic read, invalidate all the
+ * caches.
+ */
pipe_bits |= ANV_PIPE_INVALIDATE_BITS;
break;
case VK_ACCESS_MEMORY_WRITE_BIT:
+ /* Generic write, make sure all previously written things land in
+ * memory.
+ */
pipe_bits |= ANV_PIPE_FLUSH_BITS;
break;
+ case VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT:
+ /* Transitioning a buffer for conditional rendering. We'll load the
+ * content of this buffer into HW registers using the command
+ * streamer, so we need to stall the command streamer to make sure
+ * any in-flight flush operations have completed.
+ */
+ pipe_bits |= ANV_PIPE_CS_STALL_BIT;
+ break;
default:
break; /* Nothing to do */
}
VkDeviceSize offset;
};
+struct anv_xfb_binding {
+ struct anv_buffer * buffer;
+ VkDeviceSize offset;
+ VkDeviceSize size;
+};
+
#define ANV_PARAM_PUSH(offset) ((1 << 16) | (uint32_t)(offset))
#define ANV_PARAM_PUSH_OFFSET(param) ((param) & 0xffff)
VkRect2D render_area;
uint32_t restart_index;
struct anv_vertex_binding vertex_bindings[MAX_VBS];
+ bool xfb_enabled;
+ struct anv_xfb_binding xfb_bindings[MAX_XFB_BUFFERS];
VkShaderStageFlags push_constant_stages;
struct anv_push_constants * push_constants[MESA_SHADER_STAGES];
struct anv_state binding_tables[MESA_SHADER_STAGES];
*/
bool hiz_enabled;
+ bool conditional_render_enabled;
+
/**
* Array length is anv_cmd_state::pass::attachment_count. Array content is
* valid only when recording a render pass instance.
void anv_cmd_buffer_dump(struct anv_cmd_buffer *cmd_buffer);
+void anv_cmd_emit_conditional_render_predicate(struct anv_cmd_buffer *cmd_buffer);
+
enum anv_fence_type {
ANV_FENCE_TYPE_NONE = 0,
ANV_FENCE_TYPE_BO,
struct anv_pipeline_bind_map {
uint32_t surface_count;
uint32_t sampler_count;
- uint32_t image_count;
+ uint32_t image_param_count;
struct anv_pipeline_binding * surface_to_descriptor;
struct anv_pipeline_binding * sampler_to_descriptor;
const struct brw_stage_prog_data *prog_data;
uint32_t prog_data_size;
+ struct nir_xfb_info *xfb_info;
+
struct anv_pipeline_bind_map bind_map;
};
const void *constant_data, uint32_t constant_data_size,
const struct brw_stage_prog_data *prog_data,
uint32_t prog_data_size, const void *prog_data_param,
+ const struct nir_xfb_info *xfb_info,
const struct anv_pipeline_bind_map *bind_map);
void
uint32_t instance_divisor;
} vb[MAX_VBS];
+ uint8_t xfb_used;
+
bool primitive_restart;
uint32_t topology;
bool writes_stencil;
bool stencil_test_enable;
bool depth_clamp_enable;
+ bool depth_clip_enable;
bool sample_shading_enable;
bool kill_pixel;
projects / mesa.git / blobdiff