#define VG(x)
#endif
+#include "common/gen_clflush.h"
#include "common/gen_device_info.h"
#include "blorp/blorp.h"
#include "compiler/brw_compiler.h"
#include "util/macros.h"
#include "util/list.h"
+#include "util/u_atomic.h"
#include "util/u_vector.h"
-#include "util/vk_alloc.h"
+#include "vk_alloc.h"
/* Pre-declarations needed for WSI entrypoints */
struct wl_surface;
*/
#define ANV_HZ_FC_VAL 1.0f
-#define MAX_VBS 31
+#define MAX_VBS 28
#define MAX_SETS 8
#define MAX_RTS 8
#define MAX_VIEWPORTS 16
#define anv_assert(x)
#endif
+/* A multi-pointer allocator
+ *
+ * When copying data structures from the user (such as a render pass), it's
+ * common to need to allocate data for a bunch of different things. Instead
+ * of doing several allocations and having to handle all of the error checking
+ * that entails, it can be easier to do a single allocation. This struct
+ * helps facilitate that. The intended usage looks like this:
+ *
+ * ANV_MULTIALLOC(ma)
+ * anv_multialloc_add(&ma, &main_ptr, 1);
+ * anv_multialloc_add(&ma, &substruct1, substruct1Count);
+ * anv_multialloc_add(&ma, &substruct2, substruct2Count);
+ *
+ * if (!anv_multialloc_alloc(&ma, pAllocator, VK_ALLOCATION_SCOPE_FOO))
+ * return vk_error(VK_ERROR_OUT_OF_HOST_MEORY);
+ */
+struct anv_multialloc {
+ size_t size;
+ size_t align;
+
+ uint32_t ptr_count;
+ void **ptrs[8];
+};
+
+#define ANV_MULTIALLOC_INIT \
+ ((struct anv_multialloc) { 0, })
+
+#define ANV_MULTIALLOC(_name) \
+ struct anv_multialloc _name = ANV_MULTIALLOC_INIT
+
+__attribute__((always_inline))
+static inline void
+_anv_multialloc_add(struct anv_multialloc *ma,
+ void **ptr, size_t size, size_t align)
+{
+ size_t offset = align_u64(ma->size, align);
+ ma->size = offset + size;
+ ma->align = MAX2(ma->align, align);
+
+ /* Store the offset in the pointer. */
+ *ptr = (void *)(uintptr_t)offset;
+
+ assert(ma->ptr_count < ARRAY_SIZE(ma->ptrs));
+ ma->ptrs[ma->ptr_count++] = ptr;
+}
+
+#define anv_multialloc_add(_ma, _ptr, _count) \
+ _anv_multialloc_add((_ma), (void **)(_ptr), \
+ (_count) * sizeof(**(_ptr)), __alignof__(**(_ptr)))
+
+__attribute__((always_inline))
+static inline void *
+anv_multialloc_alloc(struct anv_multialloc *ma,
+ const VkAllocationCallbacks *alloc,
+ VkSystemAllocationScope scope)
+{
+ void *ptr = vk_alloc(alloc, ma->size, ma->align, scope);
+ if (!ptr)
+ return NULL;
+
+ /* Fill out each of the pointers with their final value.
+ *
+ * for (uint32_t i = 0; i < ma->ptr_count; i++)
+ * *ma->ptrs[i] = ptr + (uintptr_t)*ma->ptrs[i];
+ *
+ * Unfortunately, even though ma->ptr_count is basically guaranteed to be a
+ * constant, GCC is incapable of figuring this out and unrolling the loop
+ * so we have to give it a little help.
+ */
+ STATIC_ASSERT(ARRAY_SIZE(ma->ptrs) == 8);
+#define _ANV_MULTIALLOC_UPDATE_POINTER(_i) \
+ if ((_i) < ma->ptr_count) \
+ *ma->ptrs[_i] = ptr + (uintptr_t)*ma->ptrs[_i]
+ _ANV_MULTIALLOC_UPDATE_POINTER(0);
+ _ANV_MULTIALLOC_UPDATE_POINTER(1);
+ _ANV_MULTIALLOC_UPDATE_POINTER(2);
+ _ANV_MULTIALLOC_UPDATE_POINTER(3);
+ _ANV_MULTIALLOC_UPDATE_POINTER(4);
+ _ANV_MULTIALLOC_UPDATE_POINTER(5);
+ _ANV_MULTIALLOC_UPDATE_POINTER(6);
+ _ANV_MULTIALLOC_UPDATE_POINTER(7);
+#undef _ANV_MULTIALLOC_UPDATE_POINTER
+
+ return ptr;
+}
+
+__attribute__((always_inline))
+static inline void *
+anv_multialloc_alloc2(struct anv_multialloc *ma,
+ const VkAllocationCallbacks *parent_alloc,
+ const VkAllocationCallbacks *alloc,
+ VkSystemAllocationScope scope)
+{
+ return anv_multialloc_alloc(ma, alloc ? alloc : parent_alloc, scope);
+}
+
/**
* A dynamically growable, circular buffer. Elements are added at head and
* removed from tail. head and tail are free-running uint32_t indices and we
uint64_t size;
void *map;
- /* We need to set the WRITE flag on winsys bos so GEM will know we're
- * writing to them and synchronize uses on other rings (eg if the display
- * server uses the blitter ring).
- */
- bool is_winsys_bo;
+ /** Flags to pass to the kernel through drm_i915_exec_object2::flags */
+ uint32_t flags;
};
static inline void
bo->offset = -1;
bo->size = size;
bo->map = NULL;
- bo->is_winsys_bo = false;
+ bo->flags = 0;
}
/* Represents a lock-free linked list of "free" things. This is used by
*/
struct u_vector mmap_cleanups;
- uint32_t block_size;
-
- union anv_free_list free_list;
struct anv_block_state state;
- union anv_free_list back_free_list;
struct anv_block_state back_state;
};
-/* Block pools are backed by a fixed-size 2GB memfd */
-#define BLOCK_POOL_MEMFD_SIZE (1ul << 31)
+/* Block pools are backed by a fixed-size 1GB memfd */
+#define BLOCK_POOL_MEMFD_SIZE (1ul << 30)
/* The center of the block pool is also the middle of the memfd. This may
* change in the future if we decide differently for some reason.
void *map;
};
+#define ANV_STATE_NULL ((struct anv_state) { .alloc_size = 0 })
+
struct anv_fixed_size_state_pool {
- size_t state_size;
union anv_free_list free_list;
struct anv_block_state block;
};
#define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2 + 1)
struct anv_state_pool {
- struct anv_block_pool *block_pool;
+ struct anv_block_pool block_pool;
+
+ /* The size of blocks which will be allocated from the block pool */
+ uint32_t block_size;
+
+ /** Free list for "back" allocations */
+ union anv_free_list back_alloc_free_list;
+
struct anv_fixed_size_state_pool buckets[ANV_STATE_BUCKETS];
};
struct anv_state_stream_block;
struct anv_state_stream {
- struct anv_block_pool *block_pool;
-
- /* The current working block */
- struct anv_state_stream_block *block;
-
- /* Offset at which the current block starts */
- uint32_t start;
- /* Offset at which to allocate the next state */
- uint32_t next;
- /* Offset at which the current block ends */
- uint32_t end;
-};
+ struct anv_state_pool *state_pool;
-#define CACHELINE_SIZE 64
-#define CACHELINE_MASK 63
-
-static inline void
-anv_clflush_range(void *start, size_t size)
-{
- void *p = (void *) (((uintptr_t) start) & ~CACHELINE_MASK);
- void *end = start + size;
+ /* The size of blocks to allocate from the state pool */
+ uint32_t block_size;
- while (p < end) {
- __builtin_ia32_clflush(p);
- p += CACHELINE_SIZE;
- }
-}
+ /* Current block we're allocating from */
+ struct anv_state block;
-static inline void
-anv_flush_range(void *start, size_t size)
-{
- __builtin_ia32_mfence();
- anv_clflush_range(start, size);
-}
+ /* Offset into the current block at which to allocate the next state */
+ uint32_t next;
-static inline void
-anv_invalidate_range(void *start, size_t size)
-{
- anv_clflush_range(start, size);
- __builtin_ia32_mfence();
-}
+ /* List of all blocks allocated from this pool */
+ struct anv_state_stream_block *block_list;
+};
+/* The block_pool functions exported for testing only. The block pool should
+ * only be used via a state pool (see below).
+ */
VkResult anv_block_pool_init(struct anv_block_pool *pool,
- struct anv_device *device, uint32_t block_size);
+ struct anv_device *device,
+ uint32_t initial_size);
void anv_block_pool_finish(struct anv_block_pool *pool);
-int32_t anv_block_pool_alloc(struct anv_block_pool *pool);
-int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool);
-void anv_block_pool_free(struct anv_block_pool *pool, int32_t offset);
-void anv_state_pool_init(struct anv_state_pool *pool,
- struct anv_block_pool *block_pool);
+int32_t anv_block_pool_alloc(struct anv_block_pool *pool,
+ uint32_t block_size);
+int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool,
+ uint32_t block_size);
+
+VkResult anv_state_pool_init(struct anv_state_pool *pool,
+ struct anv_device *device,
+ uint32_t block_size);
void anv_state_pool_finish(struct anv_state_pool *pool);
struct anv_state anv_state_pool_alloc(struct anv_state_pool *pool,
- size_t state_size, size_t alignment);
+ uint32_t state_size, uint32_t alignment);
+struct anv_state anv_state_pool_alloc_back(struct anv_state_pool *pool);
void anv_state_pool_free(struct anv_state_pool *pool, struct anv_state state);
void anv_state_stream_init(struct anv_state_stream *stream,
- struct anv_block_pool *block_pool);
+ struct anv_state_pool *state_pool,
+ uint32_t block_size);
void anv_state_stream_finish(struct anv_state_stream *stream);
struct anv_state anv_state_stream_alloc(struct anv_state_stream *stream,
uint32_t size, uint32_t alignment);
gl_shader_stage stage,
unsigned per_thread_scratch);
+/** Implements a BO cache that ensures a 1-1 mapping of GEM BOs to anv_bos */
+struct anv_bo_cache {
+ struct hash_table *bo_map;
+ pthread_mutex_t mutex;
+};
+
+VkResult anv_bo_cache_init(struct anv_bo_cache *cache);
+void anv_bo_cache_finish(struct anv_bo_cache *cache);
+VkResult anv_bo_cache_alloc(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ uint64_t size, struct anv_bo **bo);
+VkResult anv_bo_cache_import(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ int fd, uint64_t size, struct anv_bo **bo);
+VkResult anv_bo_cache_export(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ struct anv_bo *bo_in, int *fd_out);
+void anv_bo_cache_release(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ struct anv_bo *bo);
+
+struct anv_memory_type {
+ /* Standard bits passed on to the client */
+ VkMemoryPropertyFlags propertyFlags;
+ uint32_t heapIndex;
+
+ /* Driver-internal book-keeping */
+ VkBufferUsageFlags valid_buffer_usage;
+};
+
+struct anv_memory_heap {
+ /* Standard bits passed on to the client */
+ VkDeviceSize size;
+ VkMemoryHeapFlags flags;
+
+ /* Driver-internal book-keeping */
+ bool supports_48bit_addresses;
+};
+
struct anv_physical_device {
VK_LOADER_DATA _loader_data;
char path[20];
const char * name;
struct gen_device_info info;
- uint64_t aperture_size;
+ /** Amount of "GPU memory" we want to advertise
+ *
+ * Clearly, this value is bogus since Intel is a UMA architecture. On
+ * gen7 platforms, we are limited by GTT size unless we want to implement
+ * fine-grained tracking and GTT splitting. On Broadwell and above we are
+ * practically unlimited. However, we will never report more than 3/4 of
+ * the total system ram to try and avoid running out of RAM.
+ */
+ bool supports_48bit_addresses;
struct brw_compiler * compiler;
struct isl_device isl_dev;
int cmd_parser_version;
+ bool has_exec_async;
uint32_t eu_total;
uint32_t subslice_total;
- uint8_t uuid[VK_UUID_SIZE];
+ struct {
+ uint32_t type_count;
+ struct anv_memory_type types[VK_MAX_MEMORY_TYPES];
+ uint32_t heap_count;
+ struct anv_memory_heap heaps[VK_MAX_MEMORY_HEAPS];
+ } memory;
+
+ uint8_t pipeline_cache_uuid[VK_UUID_SIZE];
+ uint8_t driver_uuid[VK_UUID_SIZE];
+ uint8_t device_uuid[VK_UUID_SIZE];
struct wsi_device wsi_device;
int local_fd;
VkResult anv_init_wsi(struct anv_physical_device *physical_device);
void anv_finish_wsi(struct anv_physical_device *physical_device);
+bool anv_instance_extension_supported(const char *name);
+bool anv_physical_device_extension_supported(struct anv_physical_device *dev,
+ const char *name);
+
struct anv_queue {
VK_LOADER_DATA _loader_data;
struct anv_bo_pool batch_bo_pool;
- struct anv_block_pool dynamic_state_block_pool;
- struct anv_state_pool dynamic_state_pool;
+ struct anv_bo_cache bo_cache;
- struct anv_block_pool instruction_block_pool;
+ struct anv_state_pool dynamic_state_pool;
struct anv_state_pool instruction_state_pool;
-
- struct anv_block_pool surface_state_block_pool;
struct anv_state_pool surface_state_pool;
struct anv_bo workaround_bo;
pthread_mutex_t mutex;
pthread_cond_t queue_submit;
+ bool lost;
};
static void inline
if (device->info.has_llc)
return;
- anv_flush_range(state.map, state.alloc_size);
+ gen_flush_range(state.map, state.alloc_size);
}
void anv_device_init_blorp(struct anv_device *device);
VkResult anv_device_execbuf(struct anv_device *device,
struct drm_i915_gem_execbuffer2 *execbuf,
struct anv_bo **execbuf_bos);
+VkResult anv_device_query_status(struct anv_device *device);
+VkResult anv_device_bo_busy(struct anv_device *device, struct anv_bo *bo);
+VkResult anv_device_wait(struct anv_device *device, struct anv_bo *bo,
+ int64_t timeout);
void* anv_gem_mmap(struct anv_device *device,
uint32_t gem_handle, uint64_t offset, uint64_t size, uint32_t flags);
void anv_gem_munmap(void *p, uint64_t size);
-uint32_t anv_gem_create(struct anv_device *device, size_t size);
+uint32_t anv_gem_create(struct anv_device *device, uint64_t size);
void anv_gem_close(struct anv_device *device, uint32_t gem_handle);
uint32_t anv_gem_userptr(struct anv_device *device, void *mem, size_t size);
+int anv_gem_busy(struct anv_device *device, uint32_t gem_handle);
int anv_gem_wait(struct anv_device *device, uint32_t gem_handle, int64_t *timeout_ns);
int anv_gem_execbuffer(struct anv_device *device,
struct drm_i915_gem_execbuffer2 *execbuf);
uint32_t stride, uint32_t tiling);
int anv_gem_create_context(struct anv_device *device);
int anv_gem_destroy_context(struct anv_device *device, int context);
+int anv_gem_get_context_param(int fd, int context, uint32_t param,
+ uint64_t *value);
int anv_gem_get_param(int fd, uint32_t param);
bool anv_gem_get_bit6_swizzle(int fd, uint32_t tiling);
int anv_gem_get_aperture(int fd, uint64_t *size);
+bool anv_gem_supports_48b_addresses(int fd);
+int anv_gem_gpu_get_reset_stats(struct anv_device *device,
+ uint32_t *active, uint32_t *pending);
int anv_gem_handle_to_fd(struct anv_device *device, uint32_t gem_handle);
uint32_t anv_gem_fd_to_handle(struct anv_device *device, int fd);
int anv_gem_set_caching(struct anv_device *device, uint32_t gem_handle, uint32_t caching);
VkResult anv_bo_init_new(struct anv_bo *bo, struct anv_device *device, uint64_t size);
struct anv_reloc_list {
- size_t num_relocs;
- size_t array_length;
+ uint32_t num_relocs;
+ uint32_t array_length;
struct drm_i915_gem_relocation_entry * relocs;
struct anv_bo ** reloc_bos;
};
void anv_reloc_list_finish(struct anv_reloc_list *list,
const VkAllocationCallbacks *alloc);
-uint64_t anv_reloc_list_add(struct anv_reloc_list *list,
+VkResult anv_reloc_list_add(struct anv_reloc_list *list,
const VkAllocationCallbacks *alloc,
uint32_t offset, struct anv_bo *target_bo,
uint32_t delta);
struct anv_bo bo;
/* Bytes actually consumed in this batch BO */
- size_t length;
+ uint32_t length;
struct anv_reloc_list relocs;
};
*/
VkResult (*extend_cb)(struct anv_batch *, void *);
void * user_data;
+
+ /**
+ * Current error status of the command buffer. Used to track inconsistent
+ * or incomplete command buffer states that are the consequence of run-time
+ * errors such as out of memory scenarios. We want to track this in the
+ * batch because the command buffer object is not visible to some parts
+ * of the driver.
+ */
+ VkResult status;
};
void *anv_batch_emit_dwords(struct anv_batch *batch, int num_dwords);
VkResult anv_device_submit_simple_batch(struct anv_device *device,
struct anv_batch *batch);
+static inline VkResult
+anv_batch_set_error(struct anv_batch *batch, VkResult error)
+{
+ assert(error != VK_SUCCESS);
+ if (batch->status == VK_SUCCESS)
+ batch->status = error;
+ return batch->status;
+}
+
+static inline bool
+anv_batch_has_error(struct anv_batch *batch)
+{
+ return batch->status != VK_SUCCESS;
+}
+
struct anv_address {
struct anv_bo *bo;
uint32_t offset;
VG(VALGRIND_CHECK_MEM_IS_DEFINED(dst, __anv_cmd_length(struc) * 4)); \
} while (0)
-#define anv_batch_emitn(batch, n, cmd, ...) ({ \
- void *__dst = anv_batch_emit_dwords(batch, n); \
- struct cmd __template = { \
- __anv_cmd_header(cmd), \
- .DWordLength = n - __anv_cmd_length_bias(cmd), \
- __VA_ARGS__ \
- }; \
- __anv_cmd_pack(cmd)(batch, __dst, &__template); \
- __dst; \
+#define anv_batch_emitn(batch, n, cmd, ...) ({ \
+ void *__dst = anv_batch_emit_dwords(batch, n); \
+ if (__dst) { \
+ struct cmd __template = { \
+ __anv_cmd_header(cmd), \
+ .DWordLength = n - __anv_cmd_length_bias(cmd), \
+ __VA_ARGS__ \
+ }; \
+ __anv_cmd_pack(cmd)(batch, __dst, &__template); \
+ } \
+ __dst; \
})
#define anv_batch_emit_merge(batch, dwords0, dwords1) \
\
STATIC_ASSERT(ARRAY_SIZE(dwords0) == ARRAY_SIZE(dwords1)); \
dw = anv_batch_emit_dwords((batch), ARRAY_SIZE(dwords0)); \
+ if (!dw) \
+ break; \
for (uint32_t i = 0; i < ARRAY_SIZE(dwords0); i++) \
dw[i] = (dwords0)[i] | (dwords1)[i]; \
VG(VALGRIND_CHECK_MEM_IS_DEFINED(dw, ARRAY_SIZE(dwords0) * 4));\
.IndextoMOCSTables = 1 \
}
+/* Cannonlake MOCS defines are duplicates of Skylake MOCS defines. */
+#define GEN10_MOCS (struct GEN10_MEMORY_OBJECT_CONTROL_STATE) { \
+ /* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */ \
+ .IndextoMOCSTables = 2 \
+ }
+
+#define GEN10_MOCS_PTE { \
+ /* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */ \
+ .IndextoMOCSTables = 1 \
+ }
+
struct anv_device_memory {
- struct anv_bo bo;
- uint32_t type_index;
+ struct anv_bo * bo;
+ struct anv_memory_type * type;
VkDeviceSize map_size;
void * map;
};
union {
struct {
+ VkImageLayout layout;
struct anv_image_view *image_view;
struct anv_sampler *sampler;
-
- /* Used to determine whether or not we need the surface state to have
- * the auxiliary buffer enabled.
- */
- enum isl_aux_usage aux_usage;
};
struct {
ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT | \
ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT)
+static inline enum anv_pipe_bits
+anv_pipe_flush_bits_for_access_flags(VkAccessFlags flags)
+{
+ enum anv_pipe_bits pipe_bits = 0;
+
+ unsigned b;
+ for_each_bit(b, flags) {
+ switch ((VkAccessFlagBits)(1 << b)) {
+ case VK_ACCESS_SHADER_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_DATA_CACHE_FLUSH_BIT;
+ break;
+ case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
+ break;
+ case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
+ break;
+ case VK_ACCESS_TRANSFER_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
+ pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
+ break;
+ default:
+ break; /* Nothing to do */
+ }
+ }
+
+ return pipe_bits;
+}
+
+static inline enum anv_pipe_bits
+anv_pipe_invalidate_bits_for_access_flags(VkAccessFlags flags)
+{
+ enum anv_pipe_bits pipe_bits = 0;
+
+ unsigned b;
+ for_each_bit(b, flags) {
+ switch ((VkAccessFlagBits)(1 << b)) {
+ case VK_ACCESS_INDIRECT_COMMAND_READ_BIT:
+ case VK_ACCESS_INDEX_READ_BIT:
+ case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT:
+ pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
+ break;
+ case VK_ACCESS_UNIFORM_READ_BIT:
+ pipe_bits |= ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
+ pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
+ break;
+ case VK_ACCESS_SHADER_READ_BIT:
+ case VK_ACCESS_INPUT_ATTACHMENT_READ_BIT:
+ case VK_ACCESS_TRANSFER_READ_BIT:
+ pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
+ break;
+ default:
+ break; /* Nothing to do */
+ }
+ }
+
+ return pipe_bits;
+}
+
struct anv_vertex_binding {
struct anv_buffer * buffer;
VkDeviceSize offset;
bool fast_clear;
VkClearValue clear_value;
bool clear_color_is_zero_one;
+ bool clear_color_is_zero;
};
/** State required while building cmd buffer */
*
* initialized by anv_cmd_buffer_init_batch_bo_chain()
*/
- struct u_vector bt_blocks;
+ struct u_vector bt_block_states;
uint32_t bt_next;
struct anv_reloc_list surface_relocs;
struct anv_cmd_buffer *secondary);
void anv_cmd_buffer_prepare_execbuf(struct anv_cmd_buffer *cmd_buffer);
VkResult anv_cmd_buffer_execbuf(struct anv_device *device,
- struct anv_cmd_buffer *cmd_buffer);
+ struct anv_cmd_buffer *cmd_buffer,
+ const VkSemaphore *in_semaphores,
+ uint32_t num_in_semaphores,
+ const VkSemaphore *out_semaphores,
+ uint32_t num_out_semaphores);
VkResult anv_cmd_buffer_reset(struct anv_cmd_buffer *cmd_buffer);
const struct anv_image_view *
anv_cmd_buffer_get_depth_stencil_view(const struct anv_cmd_buffer *cmd_buffer);
-struct anv_state
+VkResult
anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
uint32_t num_entries,
- uint32_t *state_offset);
+ uint32_t *state_offset,
+ struct anv_state *bt_state);
void anv_cmd_buffer_dump(struct anv_cmd_buffer *cmd_buffer);
struct anv_state state;
};
+enum anv_semaphore_type {
+ ANV_SEMAPHORE_TYPE_NONE = 0,
+ ANV_SEMAPHORE_TYPE_DUMMY,
+ ANV_SEMAPHORE_TYPE_BO,
+};
+
+struct anv_semaphore_impl {
+ enum anv_semaphore_type type;
+
+ /* A BO representing this semaphore when type == ANV_SEMAPHORE_TYPE_BO.
+ * This BO will be added to the object list on any execbuf2 calls for
+ * which this semaphore is used as a wait or signal fence. When used as
+ * a signal fence, the EXEC_OBJECT_WRITE flag will be set.
+ */
+ struct anv_bo *bo;
+};
+
+struct anv_semaphore {
+ /* Permanent semaphore state. Every semaphore has some form of permanent
+ * state (type != ANV_SEMAPHORE_TYPE_NONE). This may be a BO to fence on
+ * (for cross-process semaphores0 or it could just be a dummy for use
+ * internally.
+ */
+ struct anv_semaphore_impl permanent;
+
+ /* Temporary semaphore state. A semaphore *may* have temporary state.
+ * That state is added to the semaphore by an import operation and is reset
+ * back to ANV_SEMAPHORE_TYPE_NONE when the semaphore is waited on. A
+ * semaphore with temporary state cannot be signaled because the semaphore
+ * must already be signaled before the temporary state can be exported from
+ * the semaphore in the other process and imported here.
+ */
+ struct anv_semaphore_impl temporary;
+};
+
struct anv_shader_module {
unsigned char sha1[20];
uint32_t size;
char data[0];
};
-void anv_hash_shader(unsigned char *hash, const void *key, size_t key_size,
- struct anv_shader_module *module,
- const char *entrypoint,
- const struct anv_pipeline_layout *pipeline_layout,
- const VkSpecializationInfo *spec_info);
-
static inline gl_shader_stage
vk_to_mesa_shader_stage(VkShaderStageFlagBits vk_stage)
{
anv_shader_bin_ref(struct anv_shader_bin *shader)
{
assert(shader && shader->ref_cnt >= 1);
- __sync_fetch_and_add(&shader->ref_cnt, 1);
+ p_atomic_inc(&shader->ref_cnt);
}
static inline void
anv_shader_bin_unref(struct anv_device *device, struct anv_shader_bin *shader)
{
assert(shader && shader->ref_cnt >= 1);
- if (__sync_fetch_and_add(&shader->ref_cnt, -1) == 1)
+ if (p_atomic_dec_zero(&shader->ref_cnt))
anv_shader_bin_destroy(device, shader);
}
uint32_t dynamic_state_mask;
struct anv_dynamic_state dynamic_state;
+ struct anv_subpass * subpass;
struct anv_pipeline_layout * layout;
bool needs_data_cache;
bool writes_stencil;
bool stencil_test_enable;
bool depth_clamp_enable;
+ bool sample_shading_enable;
bool kill_pixel;
struct {
struct anv_surface aux_surface;
};
+/* Returns the number of auxiliary buffer levels attached to an image. */
+static inline uint8_t
+anv_image_aux_levels(const struct anv_image * const image)
+{
+ assert(image);
+ return image->aux_surface.isl.size > 0 ? image->aux_surface.isl.levels : 0;
+}
+
+/* Returns the number of auxiliary buffer layers attached to an image. */
+static inline uint32_t
+anv_image_aux_layers(const struct anv_image * const image,
+ const uint8_t miplevel)
+{
+ assert(image);
+
+ /* The miplevel must exist in the main buffer. */
+ assert(miplevel < image->levels);
+
+ if (miplevel >= anv_image_aux_levels(image)) {
+ /* There are no layers with auxiliary data because the miplevel has no
+ * auxiliary data.
+ */
+ return 0;
+ } else {
+ return MAX2(image->aux_surface.isl.logical_level0_px.array_len,
+ image->aux_surface.isl.logical_level0_px.depth >> miplevel);
+ }
+}
+
+static inline unsigned
+anv_fast_clear_state_entry_size(const struct anv_device *device)
+{
+ assert(device);
+ /* Entry contents:
+ * +--------------------------------------------+
+ * | clear value dword(s) | needs resolve dword |
+ * +--------------------------------------------+
+ */
+
+ /* Ensure that the needs resolve dword is in fact dword-aligned to enable
+ * GPU memcpy operations.
+ */
+ assert(device->isl_dev.ss.clear_value_size % 4 == 0);
+ return device->isl_dev.ss.clear_value_size + 4;
+}
+
/* Returns true if a HiZ-enabled depth buffer can be sampled from. */
static inline bool
anv_can_sample_with_hiz(const struct gen_device_info * const devinfo,
anv_gen8_hiz_op_resolve(struct anv_cmd_buffer *cmd_buffer,
const struct anv_image *image,
enum blorp_hiz_op op);
+void
+anv_ccs_resolve(struct anv_cmd_buffer * const cmd_buffer,
+ const struct anv_state surface_state,
+ const struct anv_image * const image,
+ const uint8_t level, const uint32_t layer_count,
+ const enum blorp_fast_clear_op op);
+
+void
+anv_image_fast_clear(struct anv_cmd_buffer *cmd_buffer,
+ const struct anv_image *image,
+ const uint32_t base_level, const uint32_t level_count,
+ const uint32_t base_layer, uint32_t layer_count);
enum isl_aux_usage
anv_layout_to_aux_usage(const struct gen_device_info * const devinfo,
const struct anv_image *image,
const VkImageAspectFlags aspects,
const VkImageLayout layout);
-static inline uint32_t
-anv_get_layerCount(const struct anv_image *image,
- const VkImageSubresourceRange *range)
-{
- return range->layerCount == VK_REMAINING_ARRAY_LAYERS ?
- image->array_size - range->baseArrayLayer : range->layerCount;
-}
+
+/* This is defined as a macro so that it works for both
+ * VkImageSubresourceRange and VkImageSubresourceLayers
+ */
+#define anv_get_layerCount(_image, _range) \
+ ((_range)->layerCount == VK_REMAINING_ARRAY_LAYERS ? \
+ (_image)->array_size - (_range)->baseArrayLayer : (_range)->layerCount)
static inline uint32_t
anv_get_levelCount(const struct anv_image *image,
VkFormat vk_format;
VkExtent3D extent; /**< Extent of VkImageViewCreateInfo::baseMipLevel. */
- /** RENDER_SURFACE_STATE when using image as a sampler surface. */
- struct anv_state sampler_surface_state;
+ /**
+ * RENDER_SURFACE_STATE when using image as a sampler surface with an image
+ * layout of SHADER_READ_ONLY_OPTIMAL or DEPTH_STENCIL_READ_ONLY_OPTIMAL.
+ */
+ enum isl_aux_usage optimal_sampler_aux_usage;
+ struct anv_state optimal_sampler_surface_state;
/**
- * RENDER_SURFACE_STATE when using image as a sampler surface with the
- * auxiliary buffer disabled.
+ * RENDER_SURFACE_STATE when using image as a sampler surface with an image
+ * layout of GENERAL.
*/
- struct anv_state no_aux_sampler_surface_state;
+ enum isl_aux_usage general_sampler_aux_usage;
+ struct anv_state general_sampler_surface_state;
/**
* RENDER_SURFACE_STATE when using image as a storage image. Separate states
uint32_t offset, uint32_t range,
uint32_t stride);
-void anv_image_view_fill_image_param(struct anv_device *device,
- struct anv_image_view *view,
- struct brw_image_param *param);
-void anv_buffer_view_fill_image_param(struct anv_device *device,
- struct anv_buffer_view *view,
- struct brw_image_param *param);
-
struct anv_sampler {
uint32_t state[4];
};
VkAttachmentReference depth_stencil_attachment;
+ uint32_t view_mask;
+
/** Subpass has a depth/stencil self-dependency */
bool has_ds_self_dep;
bool has_resolve;
};
-enum anv_subpass_usage {
- ANV_SUBPASS_USAGE_DRAW = (1 << 0),
- ANV_SUBPASS_USAGE_INPUT = (1 << 1),
- ANV_SUBPASS_USAGE_RESOLVE_SRC = (1 << 2),
- ANV_SUBPASS_USAGE_RESOLVE_DST = (1 << 3),
-};
+static inline unsigned
+anv_subpass_view_count(const struct anv_subpass *subpass)
+{
+ return MAX2(1, _mesa_bitcount(subpass->view_mask));
+}
struct anv_render_pass_attachment {
/* TODO: Consider using VkAttachmentDescription instead of storing each of
VkAttachmentLoadOp stencil_load_op;
VkImageLayout initial_layout;
VkImageLayout final_layout;
-
- /* An array, indexed by subpass id, of how the attachment will be used. */
- enum anv_subpass_usage * subpass_usage;
+ VkImageLayout first_subpass_layout;
/* The subpass id in which the attachment will be used last. */
uint32_t last_subpass_idx;
struct anv_render_pass {
uint32_t attachment_count;
uint32_t subpass_count;
- VkAttachmentReference * subpass_attachments;
- enum anv_subpass_usage * subpass_usages;
+ /* An array of subpass_count+1 flushes, one per subpass boundary */
+ enum anv_pipe_bits * subpass_flushes;
struct anv_render_pass_attachment * attachments;
struct anv_subpass subpasses[0];
};
-struct anv_query_pool_slot {
- uint64_t begin;
- uint64_t end;
- uint64_t available;
-};
+#define ANV_PIPELINE_STATISTICS_MASK 0x000007ff
struct anv_query_pool {
VkQueryType type;
+ VkQueryPipelineStatisticFlags pipeline_statistics;
+ /** Stride between slots, in bytes */
+ uint32_t stride;
+ /** Number of slots in this query pool */
uint32_t slots;
struct anv_bo bo;
};
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_query_pool, VkQueryPool)
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_render_pass, VkRenderPass)
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_sampler, VkSampler)
+ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_semaphore, VkSemaphore)
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_shader_module, VkShaderModule)
/* Gen-specific function declarations */
# define genX(x) gen9_##x
# include "anv_genX.h"
# undef genX
+# define genX(x) gen10_##x
+# include "anv_genX.h"
+# undef genX
#endif
#endif /* ANV_PRIVATE_H */