#include "dev/gen_device_info.h"
#include "blorp/blorp.h"
#include "compiler/brw_compiler.h"
+#include "util/bitset.h"
#include "util/macros.h"
#include "util/hash_table.h"
#include "util/list.h"
-#include "util/set.h"
#include "util/sparse_array.h"
#include "util/u_atomic.h"
#include "util/u_vector.h"
typedef uint32_t xcb_visualid_t;
typedef uint32_t xcb_window_t;
+struct anv_batch;
struct anv_buffer;
struct anv_buffer_view;
struct anv_image_view;
#include "common/intel_log.h"
#include "wsi_common.h"
+#define NSEC_PER_SEC 1000000000ull
+
/* anv Virtual Memory Layout
* =========================
*
};
}
+static inline void *anv_unpack_ptr(uintptr_t ptr, int bits, int *flags)
+{
+ uintptr_t mask = (1ull << bits) - 1;
+ *flags = ptr & mask;
+ return (void *) (ptr & ~mask);
+}
+
+static inline uintptr_t anv_pack_ptr(void *ptr, int bits, int flags)
+{
+ uintptr_t value = (uintptr_t) ptr;
+ uintptr_t mask = (1ull << bits) - 1;
+ return value | (mask & flags);
+}
+
#define for_each_bit(b, dword) \
for (uint32_t __dword = (dword); \
(b) = __builtin_ffs(__dword) - 1, __dword; \
*/
uint32_t index;
+ /* Index for use with util_sparse_array_free_list */
+ uint32_t free_index;
+
/* Last known offset. This value is provided by the kernel when we
* execbuf and is used as the presumed offset for the next bunch of
* relocations.
bool from_host_ptr:1;
};
-static inline void
-anv_bo_init(struct anv_bo *bo, uint32_t gem_handle, uint64_t size)
-{
- bo->gem_handle = gem_handle;
- bo->refcount = 1;
- bo->index = 0;
- bo->offset = -1;
- bo->size = size;
- bo->map = NULL;
- bo->flags = 0;
- bo->is_external = false;
- bo->is_wrapper = false;
- bo->has_fixed_address = false;
- bo->from_host_ptr = false;
-}
-
static inline struct anv_bo *
anv_bo_unwrap(struct anv_bo *bo)
{
struct anv_bo_pool {
struct anv_device *device;
- uint64_t bo_flags;
-
- void *free_list[16];
+ struct util_sparse_array_free_list free_list[16];
};
-void anv_bo_pool_init(struct anv_bo_pool *pool, struct anv_device *device,
- uint64_t bo_flags);
+void anv_bo_pool_init(struct anv_bo_pool *pool, struct anv_device *device);
void anv_bo_pool_finish(struct anv_bo_pool *pool);
-VkResult anv_bo_pool_alloc(struct anv_bo_pool *pool, struct anv_bo *bo,
- uint32_t size);
-void anv_bo_pool_free(struct anv_bo_pool *pool, const struct anv_bo *bo);
-
-struct anv_scratch_bo {
- bool exists;
- struct anv_bo bo;
-};
+VkResult anv_bo_pool_alloc(struct anv_bo_pool *pool, uint32_t size,
+ struct anv_bo **bo_out);
+void anv_bo_pool_free(struct anv_bo_pool *pool, struct anv_bo *bo);
struct anv_scratch_pool {
/* Indexed by Per-Thread Scratch Space number (the hardware value) and stage */
- struct anv_scratch_bo bos[16][MESA_SHADER_STAGES];
+ struct anv_bo *bos[16][MESA_SHADER_STAGES];
};
void anv_scratch_pool_init(struct anv_device *device,
/* Standard bits passed on to the client */
VkMemoryPropertyFlags propertyFlags;
uint32_t heapIndex;
-
- /* Driver-internal book-keeping */
- VkBufferUsageFlags valid_buffer_usage;
};
struct anv_memory_heap {
VkMemoryHeapFlags flags;
/* Driver-internal book-keeping */
- uint64_t vma_start;
- uint64_t vma_size;
- bool supports_48bit_addresses;
VkDeviceSize used;
};
struct isl_device isl_dev;
struct gen_perf_config * perf;
int cmd_parser_version;
+ bool has_softpin;
bool has_exec_async;
bool has_exec_capture;
bool has_exec_fence;
bool has_syncobj;
bool has_syncobj_wait;
bool has_context_priority;
- bool use_softpin;
bool has_context_isolation;
bool has_mem_available;
+ uint64_t gtt_size;
+
+ bool use_softpin;
bool always_use_bindless;
/** True if we can access buffers using A64 messages */
/** True if we can use bindless access for samplers */
bool has_bindless_samplers;
+ bool always_flush_cache;
+
struct anv_device_extension_table supported_extensions;
struct anv_physical_device_dispatch_table dispatch;
bool anv_physical_device_extension_supported(struct anv_physical_device *dev,
const char *name);
+struct anv_queue_submit {
+ struct anv_cmd_buffer * cmd_buffer;
+
+ uint32_t fence_count;
+ uint32_t fence_array_length;
+ struct drm_i915_gem_exec_fence * fences;
+
+ uint32_t temporary_semaphore_count;
+ uint32_t temporary_semaphore_array_length;
+ struct anv_semaphore_impl * temporary_semaphores;
+
+ /* Semaphores to be signaled with a SYNC_FD. */
+ struct anv_semaphore ** sync_fd_semaphores;
+ uint32_t sync_fd_semaphore_count;
+ uint32_t sync_fd_semaphore_array_length;
+
+ /* Allocated only with non shareable timelines. */
+ struct anv_timeline ** wait_timelines;
+ uint32_t wait_timeline_count;
+ uint32_t wait_timeline_array_length;
+ uint64_t * wait_timeline_values;
+
+ struct anv_timeline ** signal_timelines;
+ uint32_t signal_timeline_count;
+ uint32_t signal_timeline_array_length;
+ uint64_t * signal_timeline_values;
+
+ int in_fence;
+ bool need_out_fence;
+ int out_fence;
+
+ uint32_t fence_bo_count;
+ uint32_t fence_bo_array_length;
+ /* An array of struct anv_bo pointers with lower bit used as a flag to
+ * signal we will wait on that BO (see anv_(un)pack_ptr).
+ */
+ uintptr_t * fence_bos;
+
+ const VkAllocationCallbacks * alloc;
+ VkSystemAllocationScope alloc_scope;
+
+ struct anv_bo * simple_bo;
+ uint32_t simple_bo_size;
+
+ struct list_head link;
+};
+
struct anv_queue {
VK_LOADER_DATA _loader_data;
struct anv_device * device;
+ /*
+ * A list of struct anv_queue_submit to be submitted to i915.
+ */
+ struct list_head queued_submits;
+
VkDeviceQueueCreateFlags flags;
};
pthread_mutex_t vma_mutex;
struct util_vma_heap vma_lo;
struct util_vma_heap vma_hi;
- uint64_t vma_lo_available;
- uint64_t vma_hi_available;
/** List of all anv_device_memory objects */
struct list_head memory_objects;
struct anv_state_pool binding_table_pool;
struct anv_state_pool surface_state_pool;
- struct anv_bo workaround_bo;
- struct anv_bo trivial_batch_bo;
- struct anv_bo hiz_clear_bo;
+ struct anv_bo * workaround_bo;
+ struct anv_bo * trivial_batch_bo;
+ struct anv_bo * hiz_clear_bo;
struct anv_pipeline_cache default_pipeline_cache;
struct blorp_context blorp;
struct anv_scratch_pool scratch_pool;
- uint32_t default_mocs;
- uint32_t external_mocs;
-
pthread_mutex_t mutex;
pthread_cond_t queue_submit;
- bool _lost;
+ int _lost;
struct gen_batch_decode_ctx decoder_ctx;
/*
anv_mocs_for_bo(const struct anv_device *device, const struct anv_bo *bo)
{
if (bo->is_external)
- return device->external_mocs;
+ return device->isl_dev.mocs.external;
else
- return device->default_mocs;
+ return device->isl_dev.mocs.internal;
}
void anv_device_init_blorp(struct anv_device *device);
void anv_device_finish_blorp(struct anv_device *device);
+void _anv_device_set_all_queue_lost(struct anv_device *device);
VkResult _anv_device_set_lost(struct anv_device *device,
const char *file, int line,
const char *msg, ...)
anv_printflike(4, 5);
+VkResult _anv_queue_set_lost(struct anv_queue *queue,
+ const char *file, int line,
+ const char *msg, ...)
+ anv_printflike(4, 5);
#define anv_device_set_lost(dev, ...) \
_anv_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
+#define anv_queue_set_lost(queue, ...) \
+ _anv_queue_set_lost(queue, __FILE__, __LINE__, __VA_ARGS__)
static inline bool
anv_device_is_lost(struct anv_device *device)
{
- return unlikely(device->_lost);
+ return unlikely(p_atomic_read(&device->_lost));
}
-VkResult anv_device_execbuf(struct anv_device *device,
- struct drm_i915_gem_execbuffer2 *execbuf,
- struct anv_bo **execbuf_bos);
VkResult anv_device_query_status(struct anv_device *device);
VkResult anv_device_wait(struct anv_device *device, struct anv_bo *bo,
int64_t timeout);
+VkResult anv_queue_init(struct anv_device *device, struct anv_queue *queue);
+void anv_queue_finish(struct anv_queue *queue);
+
+VkResult anv_queue_execbuf_locked(struct anv_queue *queue, struct anv_queue_submit *submit);
+VkResult anv_queue_submit_simple_batch(struct anv_queue *queue,
+ struct anv_batch *batch);
+
+uint64_t anv_gettime_ns(void);
+uint64_t anv_get_absolute_timeout(uint64_t timeout);
+
void* anv_gem_mmap(struct anv_device *device,
uint32_t gem_handle, uint64_t offset, uint64_t size, uint32_t flags);
void anv_gem_munmap(void *p, uint64_t size);
bool anv_vma_alloc(struct anv_device *device, struct anv_bo *bo);
void anv_vma_free(struct anv_device *device, struct anv_bo *bo);
-VkResult anv_bo_init_new(struct anv_bo *bo, struct anv_device *device, uint64_t size);
-
struct anv_reloc_list {
uint32_t num_relocs;
uint32_t array_length;
struct drm_i915_gem_relocation_entry * relocs;
struct anv_bo ** reloc_bos;
- struct set * deps;
+ uint32_t dep_words;
+ BITSET_WORD * deps;
};
VkResult anv_reloc_list_init(struct anv_reloc_list *list,
/* Link in the anv_cmd_buffer.owned_batch_bos list */
struct list_head link;
- struct anv_bo bo;
+ struct anv_bo * bo;
/* Bytes actually consumed in this batch BO */
uint32_t length;
void anv_batch_emit_batch(struct anv_batch *batch, struct anv_batch *other);
uint64_t anv_batch_emit_reloc(struct anv_batch *batch,
void *location, struct anv_bo *bo, uint32_t offset);
-VkResult anv_device_submit_simple_batch(struct anv_device *device,
- struct anv_batch *batch);
static inline VkResult
anv_batch_set_error(struct anv_batch *batch, VkResult error)
_dst = NULL; \
}))
-/* MEMORY_OBJECT_CONTROL_STATE:
- * .GraphicsDataTypeGFDT = 0,
- * .LLCCacheabilityControlLLCCC = 0,
- * .L3CacheabilityControlL3CC = 1,
- */
-#define GEN7_MOCS 1
-
-/* MEMORY_OBJECT_CONTROL_STATE:
- * .LLCeLLCCacheabilityControlLLCCC = 0,
- * .L3CacheabilityControlL3CC = 1,
- */
-#define GEN75_MOCS 1
-
-/* MEMORY_OBJECT_CONTROL_STATE:
- * .MemoryTypeLLCeLLCCacheabilityControl = WB,
- * .TargetCache = L3DefertoPATforLLCeLLCselection,
- * .AgeforQUADLRU = 0
- */
-#define GEN8_MOCS 0x78
-
-/* MEMORY_OBJECT_CONTROL_STATE:
- * .MemoryTypeLLCeLLCCacheabilityControl = UCwithFenceifcoherentcycle,
- * .TargetCache = L3DefertoPATforLLCeLLCselection,
- * .AgeforQUADLRU = 0
- */
-#define GEN8_EXTERNAL_MOCS 0x18
-
-/* Skylake: MOCS is now an index into an array of 62 different caching
- * configurations programmed by the kernel.
- */
-
-/* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */
-#define GEN9_MOCS (2 << 1)
-
-/* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */
-#define GEN9_EXTERNAL_MOCS (1 << 1)
-
-/* Cannonlake MOCS defines are duplicates of Skylake MOCS defines. */
-#define GEN10_MOCS GEN9_MOCS
-#define GEN10_EXTERNAL_MOCS GEN9_EXTERNAL_MOCS
-
-/* Ice Lake MOCS defines are duplicates of Skylake MOCS defines. */
-#define GEN11_MOCS GEN9_MOCS
-#define GEN11_EXTERNAL_MOCS GEN9_EXTERNAL_MOCS
-
-/* TigerLake MOCS */
-#define GEN12_MOCS GEN9_MOCS
-/* TC=1/LLC Only, LeCC=1/Uncacheable, LRUM=0, L3CC=1/Uncacheable */
-#define GEN12_EXTERNAL_MOCS (3 << 1)
-
struct anv_device_memory {
struct list_head link;
/* Number of dynamic offsets used by this descriptor set */
uint16_t dynamic_offset_count;
+ /* For each shader stage, which offsets apply to that stage */
+ uint16_t stage_dynamic_offsets[MESA_SHADER_STAGES];
+
/* Size of the descriptor buffer for this descriptor set */
uint32_t descriptor_buffer_size;
struct anv_descriptor_pool *pool,
struct anv_descriptor_set *set);
+#define ANV_DESCRIPTOR_SET_NULL (UINT8_MAX - 5)
+#define ANV_DESCRIPTOR_SET_PUSH_CONSTANTS (UINT8_MAX - 4)
#define ANV_DESCRIPTOR_SET_DESCRIPTORS (UINT8_MAX - 3)
#define ANV_DESCRIPTOR_SET_NUM_WORK_GROUPS (UINT8_MAX - 2)
#define ANV_DESCRIPTOR_SET_SHADER_CONSTANTS (UINT8_MAX - 1)
#define ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS UINT8_MAX
struct anv_pipeline_binding {
- /* The descriptor set this surface corresponds to. The special value of
- * ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS indicates that the offset refers
- * to a color attachment and not a regular descriptor.
+ /** Index in the descriptor set
+ *
+ * This is a flattened index; the descriptor set layout is already taken
+ * into account.
+ */
+ uint32_t index;
+
+ /** The descriptor set this surface corresponds to.
+ *
+ * The special ANV_DESCRIPTOR_SET_* values above indicates that this
+ * binding is not a normal descriptor set but something else.
*/
uint8_t set;
- /* Binding in the descriptor set */
- uint32_t binding;
+ union {
+ /** Plane in the binding index for images */
+ uint8_t plane;
+
+ /** Input attachment index (relative to the subpass) */
+ uint8_t input_attachment_index;
+
+ /** Dynamic offset index (for dynamic UBOs and SSBOs) */
+ uint8_t dynamic_offset_index;
+ };
- /* Index in the binding */
+ /** For a storage image, whether it is write-only */
+ uint8_t write_only;
+
+ /** Pad to 64 bits so that there are no holes and we can safely memcmp
+ * assuming POD zero-initialization.
+ */
+ uint8_t pad;
+};
+
+struct anv_push_range {
+ /** Index in the descriptor set */
uint32_t index;
- /* Plane in the binding index */
- uint8_t plane;
+ /** Descriptor set index */
+ uint8_t set;
+
+ /** Dynamic offset index (for dynamic UBOs) */
+ uint8_t dynamic_offset_index;
- /* Input attachment index (relative to the subpass) */
- uint8_t input_attachment_index;
+ /** Start offset in units of 32B */
+ uint8_t start;
- /* For a storage image, whether it is write-only */
- bool write_only;
+ /** Range in units of 32B */
+ uint8_t length;
};
struct anv_pipeline_layout {
VkDeviceSize size;
};
-#define ANV_PARAM_PUSH(offset) ((1 << 16) | (uint32_t)(offset))
-#define ANV_PARAM_IS_PUSH(param) ((uint32_t)(param) >> 16 == 1)
-#define ANV_PARAM_PUSH_OFFSET(param) ((param) & 0xffff)
-
-#define ANV_PARAM_DYN_OFFSET(offset) ((2 << 16) | (uint32_t)(offset))
-#define ANV_PARAM_IS_DYN_OFFSET(param) ((uint32_t)(param) >> 16 == 2)
-#define ANV_PARAM_DYN_OFFSET_IDX(param) ((param) & 0xffff)
-
struct anv_push_constants {
- /* Push constant data provided by the client through vkPushConstants */
+ /** Push constant data provided by the client through vkPushConstants */
uint8_t client_data[MAX_PUSH_CONSTANTS_SIZE];
- /* Used for vkCmdDispatchBase */
- uint32_t base_work_group_id[3];
+ /** Dynamic offsets for dynamic UBOs and SSBOs */
+ uint32_t dynamic_offsets[MAX_DYNAMIC_BUFFERS];
+
+ struct {
+ /** Base workgroup ID
+ *
+ * Used for vkCmdDispatchBase.
+ */
+ uint32_t base_work_group_id[3];
+
+ /** Subgroup ID
+ *
+ * This is never set by software but is implicitly filled out when
+ * uploading the push constants for compute shaders.
+ */
+ uint32_t subgroup_id;
+
+ /** Pad out to a multiple of 32 bytes */
+ uint32_t pad[4];
+ } cs;
};
struct anv_dynamic_state {
struct anv_surface_state input;
VkImageLayout current_layout;
+ VkImageLayout current_stencil_layout;
VkImageAspectFlags pending_clear_aspects;
VkImageAspectFlags pending_load_aspects;
bool fast_clear;
struct anv_image_view * image_view;
};
+/** State tracking for vertex buffer flushes
+ *
+ * On Gen8-9, the VF cache only considers the bottom 32 bits of memory
+ * addresses. If you happen to have two vertex buffers which get placed
+ * exactly 4 GiB apart and use them in back-to-back draw calls, you can get
+ * collisions. In order to solve this problem, we track vertex address ranges
+ * which are live in the cache and invalidate the cache if one ever exceeds 32
+ * bits.
+ */
+struct anv_vb_cache_range {
+ /* Virtual address at which the live vertex buffer cache range starts for
+ * this vertex buffer index.
+ */
+ uint64_t start;
+
+ /* Virtual address of the byte after where vertex buffer cache range ends.
+ * This is exclusive such that end - start is the size of the range.
+ */
+ uint64_t end;
+};
+
/** State tracking for particular pipeline bind point
*
* This struct is the base struct for anv_cmd_graphics_state and
*/
struct anv_cmd_pipeline_state {
struct anv_pipeline *pipeline;
- struct anv_pipeline_layout *layout;
struct anv_descriptor_set *descriptors[MAX_SETS];
- uint32_t dynamic_offsets[MAX_DYNAMIC_BUFFERS];
-
struct anv_push_descriptor_set *push_descriptors[MAX_SETS];
};
anv_cmd_dirty_mask_t dirty;
uint32_t vb_dirty;
+ struct anv_vb_cache_range ib_bound_range;
+ struct anv_vb_cache_range ib_dirty_range;
+ struct anv_vb_cache_range vb_bound_ranges[33];
+ struct anv_vb_cache_range vb_dirty_ranges[33];
+
struct anv_dynamic_state dynamic;
struct {
struct anv_state binding_tables[MESA_SHADER_STAGES];
struct anv_state samplers[MESA_SHADER_STAGES];
+ unsigned char sampler_sha1s[MESA_SHADER_STAGES][20];
+ unsigned char surface_sha1s[MESA_SHADER_STAGES][20];
+ unsigned char push_sha1s[MESA_SHADER_STAGES][20];
+
/**
* Whether or not the gen8 PMA fix is enabled. We ensure that, at the top
* of any command buffer it is disabled by disabling it in EndCommandBuffer
* initialized by anv_cmd_buffer_init_batch_bo_chain()
*/
struct u_vector bt_block_states;
- uint32_t bt_next;
+ struct anv_state bt_next;
struct anv_reloc_list surface_relocs;
/** Last seen surface state block pool center bo offset */
void anv_cmd_buffer_add_secondary(struct anv_cmd_buffer *primary,
struct anv_cmd_buffer *secondary);
void anv_cmd_buffer_prepare_execbuf(struct anv_cmd_buffer *cmd_buffer);
-VkResult anv_cmd_buffer_execbuf(struct anv_device *device,
+VkResult anv_cmd_buffer_execbuf(struct anv_queue *queue,
struct anv_cmd_buffer *cmd_buffer,
const VkSemaphore *in_semaphores,
+ const uint64_t *in_wait_values,
uint32_t num_in_semaphores,
const VkSemaphore *out_semaphores,
+ const uint64_t *out_signal_values,
uint32_t num_out_semaphores,
VkFence fence);
* will say it's idle in this case.
*/
struct {
- struct anv_bo bo;
+ struct anv_bo *bo;
enum anv_bo_fence_state state;
} bo;
ANV_SEMAPHORE_TYPE_BO,
ANV_SEMAPHORE_TYPE_SYNC_FILE,
ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ,
+ ANV_SEMAPHORE_TYPE_TIMELINE,
+};
+
+struct anv_timeline_point {
+ struct list_head link;
+
+ uint64_t serial;
+
+ /* Number of waiter on this point, when > 0 the point should not be garbage
+ * collected.
+ */
+ int waiting;
+
+ /* BO used for synchronization. */
+ struct anv_bo *bo;
+};
+
+struct anv_timeline {
+ pthread_mutex_t mutex;
+ pthread_cond_t cond;
+
+ uint64_t highest_past;
+ uint64_t highest_pending;
+
+ struct list_head points;
+ struct list_head free_points;
};
struct anv_semaphore_impl {
* import so we don't need to bother with a userspace cache.
*/
uint32_t syncobj;
+
+ /* Non shareable timeline semaphore
+ *
+ * Used when kernel don't have support for timeline semaphores.
+ */
+ struct anv_timeline timeline;
};
};
struct anv_semaphore {
+ uint32_t refcount;
+
/* Permanent semaphore state. Every semaphore has some form of permanent
* state (type != ANV_SEMAPHORE_TYPE_NONE). This may be a BO to fence on
* (for cross-process semaphores0 or it could just be a dummy for use
__tmp &= ~(1 << (stage)))
struct anv_pipeline_bind_map {
+ unsigned char surface_sha1[20];
+ unsigned char sampler_sha1[20];
+ unsigned char push_sha1[20];
+
uint32_t surface_count;
uint32_t sampler_count;
struct anv_pipeline_binding * surface_to_descriptor;
struct anv_pipeline_binding * sampler_to_descriptor;
+
+ struct anv_push_range push_ranges[4];
};
struct anv_shader_bin_key {
VkImageUsageFlagBits usage;
uint32_t attachment;
VkImageLayout layout;
+
+ /* Used only with attachment containing stencil data. */
+ VkImageLayout stencil_layout;
};
struct anv_subpass {
VkImageLayout final_layout;
VkImageLayout first_subpass_layout;
+ VkImageLayout stencil_initial_layout;
+ VkImageLayout stencil_final_layout;
+
/* The subpass id in which the attachment will be used last. */
uint32_t last_subpass_idx;
};
projects / mesa.git / blobdiff