#define VG(x)
#endif
+#include "common/gen_clflush.h"
#include "common/gen_device_info.h"
#include "blorp/blorp.h"
#include "compiler/brw_compiler.h"
#include "util/macros.h"
#include "util/list.h"
+#include "util/u_atomic.h"
#include "util/u_vector.h"
-#include "util/vk_alloc.h"
+#include "vk_alloc.h"
/* Pre-declarations needed for WSI entrypoints */
struct wl_surface;
struct anv_buffer;
struct anv_buffer_view;
struct anv_image_view;
+struct anv_instance;
+struct anv_debug_report_callback;
struct gen_l3_config;
*/
#define ANV_HZ_FC_VAL 1.0f
-#define MAX_VBS 31
+#define MAX_VBS 28
#define MAX_SETS 8
#define MAX_RTS 8
#define MAX_VIEWPORTS 16
memcpy((dest), (src), (count) * sizeof(*(src))); \
})
+/* Mapping from anv object to VkDebugReportObjectTypeEXT. New types need
+ * to be added here in order to utilize mapping in debug/error/perf macros.
+ */
+#define REPORT_OBJECT_TYPE(o) \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_instance*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_INSTANCE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_physical_device*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_PHYSICAL_DEVICE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_device*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), const struct anv_device*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_queue*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_QUEUE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_semaphore*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_SEMAPHORE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_cmd_buffer*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_COMMAND_BUFFER_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_fence*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_FENCE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_device_memory*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_MEMORY_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_buffer*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_BUFFER_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_image*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), const struct anv_image*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_event*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_EVENT_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_query_pool*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_QUERY_POOL_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_buffer_view*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_BUFFER_VIEW_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_image_view*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_VIEW_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_shader_module*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_SHADER_MODULE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_pipeline_cache*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_CACHE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_pipeline_layout*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_LAYOUT_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_render_pass*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_RENDER_PASS_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_pipeline*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_set_layout*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_sampler*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_SAMPLER_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_pool*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_POOL_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_set*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_SET_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_framebuffer*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_FRAMEBUFFER_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_cmd_pool*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_COMMAND_POOL_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_surface*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_SURFACE_KHR_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct wsi_swapchain*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_SWAPCHAIN_KHR_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_debug_callback*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), void*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT, \
+ /* The void expression results in a compile-time error \
+ when assigning the result to something. */ \
+ (void)0)))))))))))))))))))))))))))))))
+
/* Whenever we generate an error, pass it through this function. Useful for
* debugging, where we can break on it. Only call at error site, not when
* propagating errors. Might be useful to plug in a stack trace here.
*/
-VkResult __vk_errorf(VkResult error, const char *file, int line, const char *format, ...);
+VkResult __vk_errorf(struct anv_instance *instance, const void *object,
+ VkDebugReportObjectTypeEXT type, VkResult error,
+ const char *file, int line, const char *format, ...);
#ifdef DEBUG
-#define vk_error(error) __vk_errorf(error, __FILE__, __LINE__, NULL);
-#define vk_errorf(error, format, ...) __vk_errorf(error, __FILE__, __LINE__, format, ## __VA_ARGS__);
+#define vk_error(error) __vk_errorf(NULL, NULL,\
+ VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT,\
+ error, __FILE__, __LINE__, NULL);
+#define vk_errorf(instance, obj, error, format, ...)\
+ __vk_errorf(instance, obj, REPORT_OBJECT_TYPE(obj), error,\
+ __FILE__, __LINE__, format, ## __VA_ARGS__);
#define anv_debug(format, ...) fprintf(stderr, "debug: " format, ##__VA_ARGS__)
#else
#define vk_error(error) error
-#define vk_errorf(error, format, ...) error
+#define vk_errorf(instance, obj, error, format, ...) error
#define anv_debug(format, ...)
#endif
* defined by extensions supported by that component.
*/
#define anv_debug_ignored_stype(sType) \
- anv_debug("debug: %s: ignored VkStructureType %u\n", __func__, (sType))
+ anv_debug("%s: ignored VkStructureType %u\n", __func__, (sType))
void __anv_finishme(const char *file, int line, const char *format, ...)
anv_printflike(3, 4);
-void __anv_perf_warn(const char *file, int line, const char *format, ...)
- anv_printflike(3, 4);
+void __anv_perf_warn(struct anv_instance *instance, const void *object,
+ VkDebugReportObjectTypeEXT type, const char *file,
+ int line, const char *format, ...)
+ anv_printflike(6, 7);
void anv_loge(const char *format, ...) anv_printflike(1, 2);
void anv_loge_v(const char *format, va_list va);
+void anv_debug_report(struct anv_instance *instance,
+ VkDebugReportFlagsEXT flags,
+ VkDebugReportObjectTypeEXT object_type,
+ uint64_t handle,
+ size_t location,
+ int32_t messageCode,
+ const char* pLayerPrefix,
+ const char *pMessage);
+
/**
* Print a FINISHME message, including its source location.
*/
/**
* Print a perf warning message. Set INTEL_DEBUG=perf to see these.
*/
-#define anv_perf_warn(format, ...) \
+#define anv_perf_warn(instance, obj, format, ...) \
do { \
static bool reported = false; \
if (!reported && unlikely(INTEL_DEBUG & DEBUG_PERF)) { \
- __anv_perf_warn(__FILE__, __LINE__, format, ##__VA_ARGS__); \
+ __anv_perf_warn(instance, obj, REPORT_OBJECT_TYPE(obj), __FILE__, __LINE__,\
+ format, ##__VA_ARGS__); \
reported = true; \
} \
} while (0)
#define anv_assert(x)
#endif
-/**
- * A dynamically growable, circular buffer. Elements are added at head and
- * removed from tail. head and tail are free-running uint32_t indices and we
- * only compute the modulo with size when accessing the array. This way,
- * number of bytes in the queue is always head - tail, even in case of
- * wraparound.
+/* A multi-pointer allocator
+ *
+ * When copying data structures from the user (such as a render pass), it's
+ * common to need to allocate data for a bunch of different things. Instead
+ * of doing several allocations and having to handle all of the error checking
+ * that entails, it can be easier to do a single allocation. This struct
+ * helps facilitate that. The intended usage looks like this:
+ *
+ * ANV_MULTIALLOC(ma)
+ * anv_multialloc_add(&ma, &main_ptr, 1);
+ * anv_multialloc_add(&ma, &substruct1, substruct1Count);
+ * anv_multialloc_add(&ma, &substruct2, substruct2Count);
+ *
+ * if (!anv_multialloc_alloc(&ma, pAllocator, VK_ALLOCATION_SCOPE_FOO))
+ * return vk_error(VK_ERROR_OUT_OF_HOST_MEORY);
*/
+struct anv_multialloc {
+ size_t size;
+ size_t align;
+
+ uint32_t ptr_count;
+ void **ptrs[8];
+};
+
+#define ANV_MULTIALLOC_INIT \
+ ((struct anv_multialloc) { 0, })
+
+#define ANV_MULTIALLOC(_name) \
+ struct anv_multialloc _name = ANV_MULTIALLOC_INIT
+
+__attribute__((always_inline))
+static inline void
+_anv_multialloc_add(struct anv_multialloc *ma,
+ void **ptr, size_t size, size_t align)
+{
+ size_t offset = align_u64(ma->size, align);
+ ma->size = offset + size;
+ ma->align = MAX2(ma->align, align);
+
+ /* Store the offset in the pointer. */
+ *ptr = (void *)(uintptr_t)offset;
+
+ assert(ma->ptr_count < ARRAY_SIZE(ma->ptrs));
+ ma->ptrs[ma->ptr_count++] = ptr;
+}
+
+#define anv_multialloc_add_size(_ma, _ptr, _size) \
+ _anv_multialloc_add((_ma), (void **)(_ptr), (_size), __alignof__(**(_ptr)))
+
+#define anv_multialloc_add(_ma, _ptr, _count) \
+ anv_multialloc_add_size(_ma, _ptr, (_count) * sizeof(**(_ptr)));
+
+__attribute__((always_inline))
+static inline void *
+anv_multialloc_alloc(struct anv_multialloc *ma,
+ const VkAllocationCallbacks *alloc,
+ VkSystemAllocationScope scope)
+{
+ void *ptr = vk_alloc(alloc, ma->size, ma->align, scope);
+ if (!ptr)
+ return NULL;
+
+ /* Fill out each of the pointers with their final value.
+ *
+ * for (uint32_t i = 0; i < ma->ptr_count; i++)
+ * *ma->ptrs[i] = ptr + (uintptr_t)*ma->ptrs[i];
+ *
+ * Unfortunately, even though ma->ptr_count is basically guaranteed to be a
+ * constant, GCC is incapable of figuring this out and unrolling the loop
+ * so we have to give it a little help.
+ */
+ STATIC_ASSERT(ARRAY_SIZE(ma->ptrs) == 8);
+#define _ANV_MULTIALLOC_UPDATE_POINTER(_i) \
+ if ((_i) < ma->ptr_count) \
+ *ma->ptrs[_i] = ptr + (uintptr_t)*ma->ptrs[_i]
+ _ANV_MULTIALLOC_UPDATE_POINTER(0);
+ _ANV_MULTIALLOC_UPDATE_POINTER(1);
+ _ANV_MULTIALLOC_UPDATE_POINTER(2);
+ _ANV_MULTIALLOC_UPDATE_POINTER(3);
+ _ANV_MULTIALLOC_UPDATE_POINTER(4);
+ _ANV_MULTIALLOC_UPDATE_POINTER(5);
+ _ANV_MULTIALLOC_UPDATE_POINTER(6);
+ _ANV_MULTIALLOC_UPDATE_POINTER(7);
+#undef _ANV_MULTIALLOC_UPDATE_POINTER
+
+ return ptr;
+}
+
+__attribute__((always_inline))
+static inline void *
+anv_multialloc_alloc2(struct anv_multialloc *ma,
+ const VkAllocationCallbacks *parent_alloc,
+ const VkAllocationCallbacks *alloc,
+ VkSystemAllocationScope scope)
+{
+ return anv_multialloc_alloc(ma, alloc ? alloc : parent_alloc, scope);
+}
struct anv_bo {
uint32_t gem_handle;
uint64_t size;
void *map;
- /* We need to set the WRITE flag on winsys bos so GEM will know we're
- * writing to them and synchronize uses on other rings (eg if the display
- * server uses the blitter ring).
- */
- bool is_winsys_bo;
+ /** Flags to pass to the kernel through drm_i915_exec_object2::flags */
+ uint32_t flags;
};
static inline void
bo->offset = -1;
bo->size = size;
bo->map = NULL;
- bo->is_winsys_bo = false;
+ bo->flags = 0;
}
/* Represents a lock-free linked list of "free" things. This is used by
*/
struct u_vector mmap_cleanups;
- uint32_t block_size;
-
- union anv_free_list free_list;
struct anv_block_state state;
- union anv_free_list back_free_list;
struct anv_block_state back_state;
};
void *map;
};
+#define ANV_STATE_NULL ((struct anv_state) { .alloc_size = 0 })
+
struct anv_fixed_size_state_pool {
- size_t state_size;
union anv_free_list free_list;
struct anv_block_state block;
};
#define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2 + 1)
struct anv_state_pool {
- struct anv_block_pool *block_pool;
+ struct anv_block_pool block_pool;
+
+ /* The size of blocks which will be allocated from the block pool */
+ uint32_t block_size;
+
+ /** Free list for "back" allocations */
+ union anv_free_list back_alloc_free_list;
+
struct anv_fixed_size_state_pool buckets[ANV_STATE_BUCKETS];
};
struct anv_state_stream_block;
struct anv_state_stream {
- struct anv_block_pool *block_pool;
+ struct anv_state_pool *state_pool;
- /* The current working block */
- struct anv_state_stream_block *block;
-
- /* Offset at which the current block starts */
- uint32_t start;
- /* Offset at which to allocate the next state */
- uint32_t next;
- /* Offset at which the current block ends */
- uint32_t end;
-};
-
-#define CACHELINE_SIZE 64
-#define CACHELINE_MASK 63
-
-static inline void
-anv_clflush_range(void *start, size_t size)
-{
- void *p = (void *) (((uintptr_t) start) & ~CACHELINE_MASK);
- void *end = start + size;
+ /* The size of blocks to allocate from the state pool */
+ uint32_t block_size;
- while (p < end) {
- __builtin_ia32_clflush(p);
- p += CACHELINE_SIZE;
- }
-}
+ /* Current block we're allocating from */
+ struct anv_state block;
-static inline void
-anv_flush_range(void *start, size_t size)
-{
- __builtin_ia32_mfence();
- anv_clflush_range(start, size);
-}
+ /* Offset into the current block at which to allocate the next state */
+ uint32_t next;
-static inline void
-anv_invalidate_range(void *start, size_t size)
-{
- anv_clflush_range(start, size);
- __builtin_ia32_mfence();
-}
+ /* List of all blocks allocated from this pool */
+ struct anv_state_stream_block *block_list;
+};
+/* The block_pool functions exported for testing only. The block pool should
+ * only be used via a state pool (see below).
+ */
VkResult anv_block_pool_init(struct anv_block_pool *pool,
- struct anv_device *device, uint32_t block_size);
+ struct anv_device *device,
+ uint32_t initial_size);
void anv_block_pool_finish(struct anv_block_pool *pool);
-int32_t anv_block_pool_alloc(struct anv_block_pool *pool);
-int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool);
-void anv_block_pool_free(struct anv_block_pool *pool, int32_t offset);
-void anv_state_pool_init(struct anv_state_pool *pool,
- struct anv_block_pool *block_pool);
+int32_t anv_block_pool_alloc(struct anv_block_pool *pool,
+ uint32_t block_size);
+int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool,
+ uint32_t block_size);
+
+VkResult anv_state_pool_init(struct anv_state_pool *pool,
+ struct anv_device *device,
+ uint32_t block_size);
void anv_state_pool_finish(struct anv_state_pool *pool);
struct anv_state anv_state_pool_alloc(struct anv_state_pool *pool,
- size_t state_size, size_t alignment);
+ uint32_t state_size, uint32_t alignment);
+struct anv_state anv_state_pool_alloc_back(struct anv_state_pool *pool);
void anv_state_pool_free(struct anv_state_pool *pool, struct anv_state state);
void anv_state_stream_init(struct anv_state_stream *stream,
- struct anv_block_pool *block_pool);
+ struct anv_state_pool *state_pool,
+ uint32_t block_size);
void anv_state_stream_finish(struct anv_state_stream *stream);
struct anv_state anv_state_stream_alloc(struct anv_state_stream *stream,
uint32_t size, uint32_t alignment);
gl_shader_stage stage,
unsigned per_thread_scratch);
+/** Implements a BO cache that ensures a 1-1 mapping of GEM BOs to anv_bos */
+struct anv_bo_cache {
+ struct hash_table *bo_map;
+ pthread_mutex_t mutex;
+};
+
+VkResult anv_bo_cache_init(struct anv_bo_cache *cache);
+void anv_bo_cache_finish(struct anv_bo_cache *cache);
+VkResult anv_bo_cache_alloc(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ uint64_t size, struct anv_bo **bo);
+VkResult anv_bo_cache_import(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ int fd, uint64_t size, struct anv_bo **bo);
+VkResult anv_bo_cache_export(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ struct anv_bo *bo_in, int *fd_out);
+void anv_bo_cache_release(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ struct anv_bo *bo);
+
+struct anv_memory_type {
+ /* Standard bits passed on to the client */
+ VkMemoryPropertyFlags propertyFlags;
+ uint32_t heapIndex;
+
+ /* Driver-internal book-keeping */
+ VkBufferUsageFlags valid_buffer_usage;
+};
+
+struct anv_memory_heap {
+ /* Standard bits passed on to the client */
+ VkDeviceSize size;
+ VkMemoryHeapFlags flags;
+
+ /* Driver-internal book-keeping */
+ bool supports_48bit_addresses;
+};
+
struct anv_physical_device {
VK_LOADER_DATA _loader_data;
char path[20];
const char * name;
struct gen_device_info info;
- uint64_t aperture_size;
+ /** Amount of "GPU memory" we want to advertise
+ *
+ * Clearly, this value is bogus since Intel is a UMA architecture. On
+ * gen7 platforms, we are limited by GTT size unless we want to implement
+ * fine-grained tracking and GTT splitting. On Broadwell and above we are
+ * practically unlimited. However, we will never report more than 3/4 of
+ * the total system ram to try and avoid running out of RAM.
+ */
+ bool supports_48bit_addresses;
struct brw_compiler * compiler;
struct isl_device isl_dev;
int cmd_parser_version;
+ bool has_exec_async;
+ bool has_exec_fence;
+ bool has_syncobj;
+ bool has_syncobj_wait;
uint32_t eu_total;
uint32_t subslice_total;
- uint8_t uuid[VK_UUID_SIZE];
+ struct {
+ uint32_t type_count;
+ struct anv_memory_type types[VK_MAX_MEMORY_TYPES];
+ uint32_t heap_count;
+ struct anv_memory_heap heaps[VK_MAX_MEMORY_HEAPS];
+ } memory;
+
+ uint8_t pipeline_cache_uuid[VK_UUID_SIZE];
+ uint8_t driver_uuid[VK_UUID_SIZE];
+ uint8_t device_uuid[VK_UUID_SIZE];
struct wsi_device wsi_device;
int local_fd;
};
+struct anv_debug_report_callback {
+ /* Link in the 'callbacks' list in anv_instance struct. */
+ struct list_head link;
+ VkDebugReportFlagsEXT flags;
+ PFN_vkDebugReportCallbackEXT callback;
+ void * data;
+};
+
struct anv_instance {
VK_LOADER_DATA _loader_data;
uint32_t apiVersion;
int physicalDeviceCount;
struct anv_physical_device physicalDevice;
+
+ /* VK_EXT_debug_report debug callbacks */
+ pthread_mutex_t callbacks_mutex;
+ struct list_head callbacks;
+ struct anv_debug_report_callback destroy_debug_cb;
};
VkResult anv_init_wsi(struct anv_physical_device *physical_device);
void anv_finish_wsi(struct anv_physical_device *physical_device);
+bool anv_instance_extension_supported(const char *name);
+uint32_t anv_physical_device_api_version(struct anv_physical_device *dev);
+bool anv_physical_device_extension_supported(struct anv_physical_device *dev,
+ const char *name);
+
struct anv_queue {
VK_LOADER_DATA _loader_data;
struct anv_bo_pool batch_bo_pool;
- struct anv_block_pool dynamic_state_block_pool;
- struct anv_state_pool dynamic_state_pool;
+ struct anv_bo_cache bo_cache;
- struct anv_block_pool instruction_block_pool;
+ struct anv_state_pool dynamic_state_pool;
struct anv_state_pool instruction_state_pool;
-
- struct anv_block_pool surface_state_block_pool;
struct anv_state_pool surface_state_pool;
struct anv_bo workaround_bo;
+ struct anv_bo trivial_batch_bo;
struct anv_pipeline_cache blorp_shader_cache;
struct blorp_context blorp;
if (device->info.has_llc)
return;
- anv_flush_range(state.map, state.alloc_size);
+ gen_flush_range(state.map, state.alloc_size);
}
void anv_device_init_blorp(struct anv_device *device);
struct drm_i915_gem_execbuffer2 *execbuf,
struct anv_bo **execbuf_bos);
VkResult anv_device_query_status(struct anv_device *device);
+VkResult anv_device_bo_busy(struct anv_device *device, struct anv_bo *bo);
VkResult anv_device_wait(struct anv_device *device, struct anv_bo *bo,
int64_t timeout);
void* anv_gem_mmap(struct anv_device *device,
uint32_t gem_handle, uint64_t offset, uint64_t size, uint32_t flags);
void anv_gem_munmap(void *p, uint64_t size);
-uint32_t anv_gem_create(struct anv_device *device, size_t size);
+uint32_t anv_gem_create(struct anv_device *device, uint64_t size);
void anv_gem_close(struct anv_device *device, uint32_t gem_handle);
uint32_t anv_gem_userptr(struct anv_device *device, void *mem, size_t size);
+int anv_gem_busy(struct anv_device *device, uint32_t gem_handle);
int anv_gem_wait(struct anv_device *device, uint32_t gem_handle, int64_t *timeout_ns);
int anv_gem_execbuffer(struct anv_device *device,
struct drm_i915_gem_execbuffer2 *execbuf);
uint32_t stride, uint32_t tiling);
int anv_gem_create_context(struct anv_device *device);
int anv_gem_destroy_context(struct anv_device *device, int context);
+int anv_gem_get_context_param(int fd, int context, uint32_t param,
+ uint64_t *value);
int anv_gem_get_param(int fd, uint32_t param);
bool anv_gem_get_bit6_swizzle(int fd, uint32_t tiling);
int anv_gem_get_aperture(int fd, uint64_t *size);
+bool anv_gem_supports_48b_addresses(int fd);
int anv_gem_gpu_get_reset_stats(struct anv_device *device,
uint32_t *active, uint32_t *pending);
int anv_gem_handle_to_fd(struct anv_device *device, uint32_t gem_handle);
int anv_gem_set_caching(struct anv_device *device, uint32_t gem_handle, uint32_t caching);
int anv_gem_set_domain(struct anv_device *device, uint32_t gem_handle,
uint32_t read_domains, uint32_t write_domain);
+int anv_gem_sync_file_merge(struct anv_device *device, int fd1, int fd2);
+uint32_t anv_gem_syncobj_create(struct anv_device *device, uint32_t flags);
+void anv_gem_syncobj_destroy(struct anv_device *device, uint32_t handle);
+int anv_gem_syncobj_handle_to_fd(struct anv_device *device, uint32_t handle);
+uint32_t anv_gem_syncobj_fd_to_handle(struct anv_device *device, int fd);
+int anv_gem_syncobj_export_sync_file(struct anv_device *device,
+ uint32_t handle);
+int anv_gem_syncobj_import_sync_file(struct anv_device *device,
+ uint32_t handle, int fd);
+void anv_gem_syncobj_reset(struct anv_device *device, uint32_t handle);
+bool anv_gem_supports_syncobj_wait(int fd);
+int anv_gem_syncobj_wait(struct anv_device *device,
+ uint32_t *handles, uint32_t num_handles,
+ int64_t abs_timeout_ns, bool wait_all);
VkResult anv_bo_init_new(struct anv_bo *bo, struct anv_device *device, uint64_t size);
struct anv_reloc_list {
- size_t num_relocs;
- size_t array_length;
+ uint32_t num_relocs;
+ uint32_t array_length;
struct drm_i915_gem_relocation_entry * relocs;
struct anv_bo ** reloc_bos;
};
struct anv_bo bo;
/* Bytes actually consumed in this batch BO */
- size_t length;
+ uint32_t length;
struct anv_reloc_list relocs;
};
.IndextoMOCSTables = 1 \
}
+/* Cannonlake MOCS defines are duplicates of Skylake MOCS defines. */
+#define GEN10_MOCS (struct GEN10_MEMORY_OBJECT_CONTROL_STATE) { \
+ /* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */ \
+ .IndextoMOCSTables = 2 \
+ }
+
+#define GEN10_MOCS_PTE { \
+ /* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */ \
+ .IndextoMOCSTables = 1 \
+ }
+
struct anv_device_memory {
- struct anv_bo bo;
- uint32_t type_index;
+ struct anv_bo * bo;
+ struct anv_memory_type * type;
VkDeviceSize map_size;
void * map;
};
union {
struct {
+ VkImageLayout layout;
struct anv_image_view *image_view;
struct anv_sampler *sampler;
-
- /* Used to determine whether or not we need the surface state to have
- * the auxiliary buffer enabled.
- */
- enum isl_aux_usage aux_usage;
};
struct {
/* Put this field right behind anv_descriptor_set so it fills up the
* descriptors[0] field. */
struct anv_descriptor descriptors[MAX_PUSH_DESCRIPTORS];
-
struct anv_buffer_view buffer_views[MAX_PUSH_DESCRIPTORS];
};
struct anv_descriptor_template_entry entries[0];
};
+size_t
+anv_descriptor_set_binding_layout_get_hw_size(const struct anv_descriptor_set_binding_layout *binding);
+
size_t
anv_descriptor_set_layout_size(const struct anv_descriptor_set_layout *layout);
uint8_t set;
/* Binding in the descriptor set */
- uint8_t binding;
+ uint32_t binding;
/* Index in the binding */
- uint8_t index;
+ uint32_t index;
+
+ /* Plane in the binding index */
+ uint8_t plane;
/* Input attachment index (relative to the subpass) */
uint8_t input_attachment_index;
ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT | \
ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT)
+static inline enum anv_pipe_bits
+anv_pipe_flush_bits_for_access_flags(VkAccessFlags flags)
+{
+ enum anv_pipe_bits pipe_bits = 0;
+
+ unsigned b;
+ for_each_bit(b, flags) {
+ switch ((VkAccessFlagBits)(1 << b)) {
+ case VK_ACCESS_SHADER_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_DATA_CACHE_FLUSH_BIT;
+ break;
+ case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
+ break;
+ case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
+ break;
+ case VK_ACCESS_TRANSFER_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
+ pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
+ break;
+ default:
+ break; /* Nothing to do */
+ }
+ }
+
+ return pipe_bits;
+}
+
+static inline enum anv_pipe_bits
+anv_pipe_invalidate_bits_for_access_flags(VkAccessFlags flags)
+{
+ enum anv_pipe_bits pipe_bits = 0;
+
+ unsigned b;
+ for_each_bit(b, flags) {
+ switch ((VkAccessFlagBits)(1 << b)) {
+ case VK_ACCESS_INDIRECT_COMMAND_READ_BIT:
+ case VK_ACCESS_INDEX_READ_BIT:
+ case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT:
+ pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
+ break;
+ case VK_ACCESS_UNIFORM_READ_BIT:
+ pipe_bits |= ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
+ pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
+ break;
+ case VK_ACCESS_SHADER_READ_BIT:
+ case VK_ACCESS_INPUT_ATTACHMENT_READ_BIT:
+ case VK_ACCESS_TRANSFER_READ_BIT:
+ pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
+ break;
+ default:
+ break; /* Nothing to do */
+ }
+ }
+
+ return pipe_bits;
+}
+
+#define VK_IMAGE_ASPECT_ANY_COLOR_BIT ( \
+ VK_IMAGE_ASPECT_COLOR_BIT | \
+ VK_IMAGE_ASPECT_PLANE_0_BIT_KHR | \
+ VK_IMAGE_ASPECT_PLANE_1_BIT_KHR | \
+ VK_IMAGE_ASPECT_PLANE_2_BIT_KHR)
+#define VK_IMAGE_ASPECT_PLANES_BITS ( \
+ VK_IMAGE_ASPECT_PLANE_0_BIT_KHR | \
+ VK_IMAGE_ASPECT_PLANE_1_BIT_KHR | \
+ VK_IMAGE_ASPECT_PLANE_2_BIT_KHR)
+
struct anv_vertex_binding {
struct anv_buffer * buffer;
VkDeviceSize offset;
};
+#define ANV_PARAM_PUSH(offset) ((1 << 16) | (uint32_t)(offset))
+#define ANV_PARAM_PUSH_OFFSET(param) ((param) & 0xffff)
+
struct anv_push_constants {
/* Current allocated size of this push constants data structure.
* Because a decent chunk of it may not be used (images on SKL, for
/* Push constant data provided by the client through vkPushConstants */
uint8_t client_data[MAX_PUSH_CONSTANTS_SIZE];
- /* Our hardware only provides zero-based vertex and instance id so, in
- * order to satisfy the vulkan requirements, we may have to push one or
- * both of these into the shader.
- */
- uint32_t base_vertex;
- uint32_t base_instance;
-
/* Image data for image_load_store on pre-SKL */
struct brw_image_param images[MAX_IMAGES];
};
const struct anv_dynamic_state *src,
uint32_t copy_mask);
+struct anv_surface_state {
+ struct anv_state state;
+ /** Address of the surface referred to by this state
+ *
+ * This address is relative to the start of the BO.
+ */
+ uint64_t address;
+ /* Address of the aux surface, if any
+ *
+ * This field is 0 if and only if no aux surface exists.
+ *
+ * This address is relative to the start of the BO. On gen7, the bottom 12
+ * bits of this address include extra aux information.
+ */
+ uint64_t aux_address;
+};
+
/**
* Attachment state when recording a renderpass instance.
*
struct anv_attachment_state {
enum isl_aux_usage aux_usage;
enum isl_aux_usage input_aux_usage;
- struct anv_state color_rt_state;
- struct anv_state input_att_state;
+ struct anv_surface_state color;
+ struct anv_surface_state input;
VkImageLayout current_layout;
VkImageAspectFlags pending_clear_aspects;
bool fast_clear;
VkClearValue clear_value;
bool clear_color_is_zero_one;
+ bool clear_color_is_zero;
};
/** State required while building cmd buffer */
struct anv_dynamic_state dynamic;
bool need_query_wa;
- struct anv_push_descriptor_set push_descriptor;
+ struct anv_push_descriptor_set * push_descriptors[MAX_SETS];
/**
* Whether or not the gen8 PMA fix is enabled. We ensure that, at the top
*
* initialized by anv_cmd_buffer_init_batch_bo_chain()
*/
- struct u_vector bt_blocks;
+ struct u_vector bt_block_states;
uint32_t bt_next;
struct anv_reloc_list surface_relocs;
struct anv_cmd_buffer *secondary);
void anv_cmd_buffer_prepare_execbuf(struct anv_cmd_buffer *cmd_buffer);
VkResult anv_cmd_buffer_execbuf(struct anv_device *device,
- struct anv_cmd_buffer *cmd_buffer);
+ struct anv_cmd_buffer *cmd_buffer,
+ const VkSemaphore *in_semaphores,
+ uint32_t num_in_semaphores,
+ const VkSemaphore *out_semaphores,
+ uint32_t num_out_semaphores,
+ VkFence fence);
VkResult anv_cmd_buffer_reset(struct anv_cmd_buffer *cmd_buffer);
void anv_cmd_buffer_dump(struct anv_cmd_buffer *cmd_buffer);
-enum anv_fence_state {
+enum anv_fence_type {
+ ANV_FENCE_TYPE_NONE = 0,
+ ANV_FENCE_TYPE_BO,
+ ANV_FENCE_TYPE_SYNCOBJ,
+};
+
+enum anv_bo_fence_state {
/** Indicates that this is a new (or newly reset fence) */
- ANV_FENCE_STATE_RESET,
+ ANV_BO_FENCE_STATE_RESET,
/** Indicates that this fence has been submitted to the GPU but is still
* (as far as we know) in use by the GPU.
*/
- ANV_FENCE_STATE_SUBMITTED,
+ ANV_BO_FENCE_STATE_SUBMITTED,
+
+ ANV_BO_FENCE_STATE_SIGNALED,
+};
+
+struct anv_fence_impl {
+ enum anv_fence_type type;
- ANV_FENCE_STATE_SIGNALED,
+ union {
+ /** Fence implementation for BO fences
+ *
+ * These fences use a BO and a set of CPU-tracked state flags. The BO
+ * is added to the object list of the last execbuf call in a QueueSubmit
+ * and is marked EXEC_WRITE. The state flags track when the BO has been
+ * submitted to the kernel. We need to do this because Vulkan lets you
+ * wait on a fence that has not yet been submitted and I915_GEM_BUSY
+ * will say it's idle in this case.
+ */
+ struct {
+ struct anv_bo bo;
+ enum anv_bo_fence_state state;
+ } bo;
+
+ /** DRM syncobj handle for syncobj-based fences */
+ uint32_t syncobj;
+ };
};
struct anv_fence {
- struct anv_bo bo;
- struct drm_i915_gem_execbuffer2 execbuf;
- struct drm_i915_gem_exec_object2 exec2_objects[1];
- enum anv_fence_state state;
+ /* Permanent fence state. Every fence has some form of permanent state
+ * (type != ANV_SEMAPHORE_TYPE_NONE). This may be a BO to fence on (for
+ * cross-process fences) or it could just be a dummy for use internally.
+ */
+ struct anv_fence_impl permanent;
+
+ /* Temporary fence state. A fence *may* have temporary state. That state
+ * is added to the fence by an import operation and is reset back to
+ * ANV_SEMAPHORE_TYPE_NONE when the fence is reset. A fence with temporary
+ * state cannot be signaled because the fence must already be signaled
+ * before the temporary state can be exported from the fence in the other
+ * process and imported here.
+ */
+ struct anv_fence_impl temporary;
};
struct anv_event {
struct anv_state state;
};
+enum anv_semaphore_type {
+ ANV_SEMAPHORE_TYPE_NONE = 0,
+ ANV_SEMAPHORE_TYPE_DUMMY,
+ ANV_SEMAPHORE_TYPE_BO,
+ ANV_SEMAPHORE_TYPE_SYNC_FILE,
+ ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ,
+};
+
+struct anv_semaphore_impl {
+ enum anv_semaphore_type type;
+
+ union {
+ /* A BO representing this semaphore when type == ANV_SEMAPHORE_TYPE_BO.
+ * This BO will be added to the object list on any execbuf2 calls for
+ * which this semaphore is used as a wait or signal fence. When used as
+ * a signal fence, the EXEC_OBJECT_WRITE flag will be set.
+ */
+ struct anv_bo *bo;
+
+ /* The sync file descriptor when type == ANV_SEMAPHORE_TYPE_SYNC_FILE.
+ * If the semaphore is in the unsignaled state due to either just being
+ * created or because it has been used for a wait, fd will be -1.
+ */
+ int fd;
+
+ /* Sync object handle when type == ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ.
+ * Unlike GEM BOs, DRM sync objects aren't deduplicated by the kernel on
+ * import so we don't need to bother with a userspace cache.
+ */
+ uint32_t syncobj;
+ };
+};
+
+struct anv_semaphore {
+ /* Permanent semaphore state. Every semaphore has some form of permanent
+ * state (type != ANV_SEMAPHORE_TYPE_NONE). This may be a BO to fence on
+ * (for cross-process semaphores0 or it could just be a dummy for use
+ * internally.
+ */
+ struct anv_semaphore_impl permanent;
+
+ /* Temporary semaphore state. A semaphore *may* have temporary state.
+ * That state is added to the semaphore by an import operation and is reset
+ * back to ANV_SEMAPHORE_TYPE_NONE when the semaphore is waited on. A
+ * semaphore with temporary state cannot be signaled because the semaphore
+ * must already be signaled before the temporary state can be exported from
+ * the semaphore in the other process and imported here.
+ */
+ struct anv_semaphore_impl temporary;
+};
+
+void anv_semaphore_reset_temporary(struct anv_device *device,
+ struct anv_semaphore *semaphore);
+
struct anv_shader_module {
unsigned char sha1[20];
uint32_t size;
char data[0];
};
-void anv_hash_shader(unsigned char *hash, const void *key, size_t key_size,
- struct anv_shader_module *module,
- const char *entrypoint,
- const struct anv_pipeline_layout *pipeline_layout,
- const VkSpecializationInfo *spec_info);
-
static inline gl_shader_stage
vk_to_mesa_shader_stage(VkShaderStageFlagBits vk_stage)
{
uint32_t prog_data_size;
struct anv_pipeline_bind_map bind_map;
-
- /* Prog data follows, then params, then the key, all aligned to 8-bytes */
};
struct anv_shader_bin *
anv_shader_bin_ref(struct anv_shader_bin *shader)
{
assert(shader && shader->ref_cnt >= 1);
- __sync_fetch_and_add(&shader->ref_cnt, 1);
+ p_atomic_inc(&shader->ref_cnt);
}
static inline void
anv_shader_bin_unref(struct anv_device *device, struct anv_shader_bin *shader)
{
assert(shader && shader->ref_cnt >= 1);
- if (__sync_fetch_and_add(&shader->ref_cnt, -1) == 1)
+ if (p_atomic_dec_zero(&shader->ref_cnt))
anv_shader_bin_destroy(device, shader);
}
uint32_t dynamic_state_mask;
struct anv_dynamic_state dynamic_state;
+ struct anv_subpass * subpass;
struct anv_pipeline_layout * layout;
bool needs_data_cache;
const char *entrypoint,
const VkSpecializationInfo *spec_info);
-struct anv_format {
+struct anv_format_plane {
enum isl_format isl_format:16;
struct isl_swizzle swizzle;
+
+ /* Whether this plane contains chroma channels */
+ bool has_chroma;
+
+ /* For downscaling of YUV planes */
+ uint8_t denominator_scales[2];
+
+ /* How to map sampled ycbcr planes to a single 4 component element. */
+ struct isl_swizzle ycbcr_swizzle;
};
-struct anv_format
-anv_get_format(const struct gen_device_info *devinfo, VkFormat format,
- VkImageAspectFlags aspect, VkImageTiling tiling);
+
+struct anv_format {
+ struct anv_format_plane planes[3];
+ uint8_t n_planes;
+ bool can_ycbcr;
+};
+
+static inline uint32_t
+anv_image_aspect_to_plane(VkImageAspectFlags image_aspects,
+ VkImageAspectFlags aspect_mask)
+{
+ switch (aspect_mask) {
+ case VK_IMAGE_ASPECT_COLOR_BIT:
+ case VK_IMAGE_ASPECT_DEPTH_BIT:
+ case VK_IMAGE_ASPECT_PLANE_0_BIT_KHR:
+ return 0;
+ case VK_IMAGE_ASPECT_STENCIL_BIT:
+ if ((image_aspects & VK_IMAGE_ASPECT_DEPTH_BIT) == 0)
+ return 0;
+ /* Fall-through */
+ case VK_IMAGE_ASPECT_PLANE_1_BIT_KHR:
+ return 1;
+ case VK_IMAGE_ASPECT_PLANE_2_BIT_KHR:
+ return 2;
+ default:
+ /* Purposefully assert with depth/stencil aspects. */
+ unreachable("invalid image aspect");
+ }
+}
+
+static inline uint32_t
+anv_image_aspect_get_planes(VkImageAspectFlags aspect_mask)
+{
+ uint32_t planes = 0;
+
+ if (aspect_mask & (VK_IMAGE_ASPECT_COLOR_BIT |
+ VK_IMAGE_ASPECT_DEPTH_BIT |
+ VK_IMAGE_ASPECT_STENCIL_BIT |
+ VK_IMAGE_ASPECT_PLANE_0_BIT_KHR))
+ planes++;
+ if (aspect_mask & VK_IMAGE_ASPECT_PLANE_1_BIT_KHR)
+ planes++;
+ if (aspect_mask & VK_IMAGE_ASPECT_PLANE_2_BIT_KHR)
+ planes++;
+
+ return planes;
+}
+
+static inline VkImageAspectFlags
+anv_plane_to_aspect(VkImageAspectFlags image_aspects,
+ uint32_t plane)
+{
+ if (image_aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT) {
+ if (_mesa_bitcount(image_aspects) > 1)
+ return VK_IMAGE_ASPECT_PLANE_0_BIT_KHR << plane;
+ return VK_IMAGE_ASPECT_COLOR_BIT;
+ }
+ if (image_aspects & VK_IMAGE_ASPECT_DEPTH_BIT)
+ return VK_IMAGE_ASPECT_DEPTH_BIT << plane;
+ assert(image_aspects == VK_IMAGE_ASPECT_STENCIL_BIT);
+ return VK_IMAGE_ASPECT_STENCIL_BIT;
+}
+
+#define anv_foreach_image_aspect_bit(b, image, aspects) \
+ for_each_bit(b, anv_image_expand_aspects(image, aspects))
+
+const struct anv_format *
+anv_get_format(VkFormat format);
+
+static inline uint32_t
+anv_get_format_planes(VkFormat vk_format)
+{
+ const struct anv_format *format = anv_get_format(vk_format);
+
+ return format != NULL ? format->n_planes : 0;
+}
+
+struct anv_format_plane
+anv_get_format_plane(const struct gen_device_info *devinfo, VkFormat vk_format,
+ VkImageAspectFlags aspect, VkImageTiling tiling);
static inline enum isl_format
anv_get_isl_format(const struct gen_device_info *devinfo, VkFormat vk_format,
VkImageAspectFlags aspect, VkImageTiling tiling)
{
- return anv_get_format(devinfo, vk_format, aspect, tiling).isl_format;
+ return anv_get_format_plane(devinfo, vk_format, aspect, tiling).isl_format;
}
static inline struct isl_swizzle
* of the actual surface formats.
*/
VkFormat vk_format;
+ const struct anv_format *format;
+
VkImageAspectFlags aspects;
VkExtent3D extent;
uint32_t levels;
uint32_t array_size;
uint32_t samples; /**< VkImageCreateInfo::samples */
+ uint32_t n_planes;
VkImageUsageFlags usage; /**< Superset of VkImageCreateInfo::usage. */
VkImageTiling tiling; /** VkImageCreateInfo::tiling */
VkDeviceSize size;
uint32_t alignment;
- /* Set when bound */
- struct anv_bo *bo;
- VkDeviceSize offset;
+ /* Whether the image is made of several underlying buffer objects rather a
+ * single one with different offsets.
+ */
+ bool disjoint;
/**
* Image subsurfaces
*
- * For each foo, anv_image::foo_surface is valid if and only if
- * anv_image::aspects has a foo aspect.
+ * For each foo, anv_image::planes[x].surface is valid if and only if
+ * anv_image::aspects has a x aspect. Refer to anv_image_aspect_to_plane()
+ * to figure the number associated with a given aspect.
*
* The hardware requires that the depth buffer and stencil buffer be
* separate surfaces. From Vulkan's perspective, though, depth and stencil
* reside in the same VkImage. To satisfy both the hardware and Vulkan, we
* allocate the depth and stencil buffers as separate surfaces in the same
* bo.
+ *
+ * Memory layout :
+ *
+ * -----------------------
+ * | surface0 | /|\
+ * ----------------------- |
+ * | shadow surface0 | |
+ * ----------------------- | Plane 0
+ * | aux surface0 | |
+ * ----------------------- |
+ * | fast clear colors0 | \|/
+ * -----------------------
+ * | surface1 | /|\
+ * ----------------------- |
+ * | shadow surface1 | |
+ * ----------------------- | Plane 1
+ * | aux surface1 | |
+ * ----------------------- |
+ * | fast clear colors1 | \|/
+ * -----------------------
+ * | ... |
+ * | |
+ * -----------------------
*/
- union {
- struct anv_surface color_surface;
+ struct {
+ /**
+ * Offset of the entire plane (whenever the image is disjoint this is
+ * set to 0).
+ */
+ uint32_t offset;
+
+ VkDeviceSize size;
+ uint32_t alignment;
+
+ struct anv_surface surface;
+
+ /**
+ * A surface which shadows the main surface and may have different
+ * tiling. This is used for sampling using a tiling that isn't supported
+ * for other operations.
+ */
+ struct anv_surface shadow_surface;
+
+ /**
+ * For color images, this is the aux usage for this image when not used
+ * as a color attachment.
+ *
+ * For depth/stencil images, this is set to ISL_AUX_USAGE_HIZ if the
+ * image has a HiZ buffer.
+ */
+ enum isl_aux_usage aux_usage;
+
+ struct anv_surface aux_surface;
+
+ /**
+ * Offset of the fast clear state (used to compute the
+ * fast_clear_state_offset of the following planes).
+ */
+ uint32_t fast_clear_state_offset;
+
+ /**
+ * BO associated with this plane, set when bound.
+ */
+ struct anv_bo *bo;
+ VkDeviceSize bo_offset;
+ } planes[3];
+};
+
+/* Returns the number of auxiliary buffer levels attached to an image. */
+static inline uint8_t
+anv_image_aux_levels(const struct anv_image * const image,
+ VkImageAspectFlagBits aspect)
+{
+ uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
+ return image->planes[plane].aux_surface.isl.size > 0 ?
+ image->planes[plane].aux_surface.isl.levels : 0;
+}
- struct {
- struct anv_surface depth_surface;
- struct anv_surface stencil_surface;
- };
- };
+/* Returns the number of auxiliary buffer layers attached to an image. */
+static inline uint32_t
+anv_image_aux_layers(const struct anv_image * const image,
+ VkImageAspectFlagBits aspect,
+ const uint8_t miplevel)
+{
+ assert(image);
- /**
- * For color images, this is the aux usage for this image when not used as a
- * color attachment.
- *
- * For depth/stencil images, this is set to ISL_AUX_USAGE_HIZ if the image
- * has a HiZ buffer.
+ /* The miplevel must exist in the main buffer. */
+ assert(miplevel < image->levels);
+
+ if (miplevel >= anv_image_aux_levels(image, aspect)) {
+ /* There are no layers with auxiliary data because the miplevel has no
+ * auxiliary data.
+ */
+ return 0;
+ } else {
+ uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
+ return MAX2(image->planes[plane].aux_surface.isl.logical_level0_px.array_len,
+ image->planes[plane].aux_surface.isl.logical_level0_px.depth >> miplevel);
+ }
+}
+
+static inline unsigned
+anv_fast_clear_state_entry_size(const struct anv_device *device)
+{
+ assert(device);
+ /* Entry contents:
+ * +--------------------------------------------+
+ * | clear value dword(s) | needs resolve dword |
+ * +--------------------------------------------+
*/
- enum isl_aux_usage aux_usage;
- struct anv_surface aux_surface;
-};
+ /* Ensure that the needs resolve dword is in fact dword-aligned to enable
+ * GPU memcpy operations.
+ */
+ assert(device->isl_dev.ss.clear_value_size % 4 == 0);
+ return device->isl_dev.ss.clear_value_size + 4;
+}
/* Returns true if a HiZ-enabled depth buffer can be sampled from. */
static inline bool
anv_can_sample_with_hiz(const struct gen_device_info * const devinfo,
- const VkImageAspectFlags aspect_mask,
- const uint32_t samples)
+ const struct anv_image *image)
{
- /* Validate the inputs. */
- assert(devinfo && aspect_mask && samples);
- return devinfo->gen >= 8 && (aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) &&
- samples == 1;
+ if (!(image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT))
+ return false;
+
+ if (devinfo->gen < 8)
+ return false;
+
+ return image->samples == 1;
}
void
anv_gen8_hiz_op_resolve(struct anv_cmd_buffer *cmd_buffer,
const struct anv_image *image,
enum blorp_hiz_op op);
+void
+anv_ccs_resolve(struct anv_cmd_buffer * const cmd_buffer,
+ const struct anv_state surface_state,
+ const struct anv_image * const image,
+ VkImageAspectFlagBits aspect,
+ const uint8_t level, const uint32_t layer_count,
+ const enum blorp_fast_clear_op op);
+
+void
+anv_image_fast_clear(struct anv_cmd_buffer *cmd_buffer,
+ const struct anv_image *image,
+ VkImageAspectFlagBits aspect,
+ const uint32_t base_level, const uint32_t level_count,
+ const uint32_t base_layer, uint32_t layer_count);
+
+void
+anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
+ const struct anv_image *image,
+ uint32_t base_level, uint32_t level_count,
+ uint32_t base_layer, uint32_t layer_count);
enum isl_aux_usage
anv_layout_to_aux_usage(const struct gen_device_info * const devinfo,
const struct anv_image *image,
- const VkImageAspectFlags aspects,
+ const VkImageAspectFlagBits aspect,
const VkImageLayout layout);
/* This is defined as a macro so that it works for both
image->levels - range->baseMipLevel : range->levelCount;
}
+static inline VkImageAspectFlags
+anv_image_expand_aspects(const struct anv_image *image,
+ VkImageAspectFlags aspects)
+{
+ /* If the underlying image has color plane aspects and
+ * VK_IMAGE_ASPECT_COLOR_BIT has been requested, then return the aspects of
+ * the underlying image. */
+ if ((image->aspects & VK_IMAGE_ASPECT_PLANES_BITS) != 0 &&
+ aspects == VK_IMAGE_ASPECT_COLOR_BIT)
+ return image->aspects;
+
+ return aspects;
+}
+
+static inline bool
+anv_image_aspects_compatible(VkImageAspectFlags aspects1,
+ VkImageAspectFlags aspects2)
+{
+ if (aspects1 == aspects2)
+ return true;
+
+ /* Only 1 color aspects are compatibles. */
+ if ((aspects1 & VK_IMAGE_ASPECT_ANY_COLOR_BIT) != 0 &&
+ (aspects2 & VK_IMAGE_ASPECT_ANY_COLOR_BIT) != 0 &&
+ _mesa_bitcount(aspects1) == _mesa_bitcount(aspects2))
+ return true;
+
+ return false;
+}
struct anv_image_view {
const struct anv_image *image; /**< VkImageViewCreateInfo::image */
- struct anv_bo *bo;
- uint32_t offset; /**< Offset into bo. */
-
- struct isl_view isl;
VkImageAspectFlags aspect_mask;
VkFormat vk_format;
VkExtent3D extent; /**< Extent of VkImageViewCreateInfo::baseMipLevel. */
- /** RENDER_SURFACE_STATE when using image as a sampler surface. */
- struct anv_state sampler_surface_state;
-
- /**
- * RENDER_SURFACE_STATE when using image as a sampler surface with the
- * auxiliary buffer disabled.
- */
- struct anv_state no_aux_sampler_surface_state;
-
- /**
- * RENDER_SURFACE_STATE when using image as a storage image. Separate states
- * for write-only and readable, using the real format for write-only and the
- * lowered format for readable.
- */
- struct anv_state storage_surface_state;
- struct anv_state writeonly_storage_surface_state;
-
- struct brw_image_param storage_image_param;
-};
+ unsigned n_planes;
+ struct {
+ uint32_t image_plane;
+
+ struct isl_view isl;
+
+ /**
+ * RENDER_SURFACE_STATE when using image as a sampler surface with an
+ * image layout of SHADER_READ_ONLY_OPTIMAL or
+ * DEPTH_STENCIL_READ_ONLY_OPTIMAL.
+ */
+ struct anv_surface_state optimal_sampler_surface_state;
+
+ /**
+ * RENDER_SURFACE_STATE when using image as a sampler surface with an
+ * image layout of GENERAL.
+ */
+ struct anv_surface_state general_sampler_surface_state;
+
+ /**
+ * RENDER_SURFACE_STATE when using image as a storage image. Separate
+ * states for write-only and readable, using the real format for
+ * write-only and the lowered format for readable.
+ */
+ struct anv_surface_state storage_surface_state;
+ struct anv_surface_state writeonly_storage_surface_state;
+
+ struct brw_image_param storage_image_param;
+ } planes[3];
+};
+
+enum anv_image_view_state_flags {
+ ANV_IMAGE_VIEW_STATE_STORAGE_WRITE_ONLY = (1 << 0),
+ ANV_IMAGE_VIEW_STATE_TEXTURE_OPTIMAL = (1 << 1),
+};
+
+void anv_image_fill_surface_state(struct anv_device *device,
+ const struct anv_image *image,
+ VkImageAspectFlagBits aspect,
+ const struct isl_view *view,
+ isl_surf_usage_flags_t view_usage,
+ enum isl_aux_usage aux_usage,
+ const union isl_color_value *clear_color,
+ enum anv_image_view_state_flags flags,
+ struct anv_surface_state *state_inout,
+ struct brw_image_param *image_param_out);
struct anv_image_create_info {
const VkImageCreateInfo *vk_info;
uint32_t offset, uint32_t range,
uint32_t stride);
-void anv_image_view_fill_image_param(struct anv_device *device,
- struct anv_image_view *view,
- struct brw_image_param *param);
-void anv_buffer_view_fill_image_param(struct anv_device *device,
- struct anv_buffer_view *view,
- struct brw_image_param *param);
+
+struct anv_ycbcr_conversion {
+ const struct anv_format * format;
+ VkSamplerYcbcrModelConversionKHR ycbcr_model;
+ VkSamplerYcbcrRangeKHR ycbcr_range;
+ VkComponentSwizzle mapping[4];
+ VkChromaLocationKHR chroma_offsets[2];
+ VkFilter chroma_filter;
+ bool chroma_reconstruction;
+};
struct anv_sampler {
- uint32_t state[4];
+ uint32_t state[3][4];
+ uint32_t n_planes;
+ struct anv_ycbcr_conversion *conversion;
};
struct anv_framebuffer {
VkAttachmentReference depth_stencil_attachment;
+ uint32_t view_mask;
+
/** Subpass has a depth/stencil self-dependency */
bool has_ds_self_dep;
bool has_resolve;
};
-enum anv_subpass_usage {
- ANV_SUBPASS_USAGE_DRAW = (1 << 0),
- ANV_SUBPASS_USAGE_INPUT = (1 << 1),
- ANV_SUBPASS_USAGE_RESOLVE_SRC = (1 << 2),
- ANV_SUBPASS_USAGE_RESOLVE_DST = (1 << 3),
-};
+static inline unsigned
+anv_subpass_view_count(const struct anv_subpass *subpass)
+{
+ return MAX2(1, _mesa_bitcount(subpass->view_mask));
+}
struct anv_render_pass_attachment {
/* TODO: Consider using VkAttachmentDescription instead of storing each of
VkAttachmentLoadOp stencil_load_op;
VkImageLayout initial_layout;
VkImageLayout final_layout;
-
- /* An array, indexed by subpass id, of how the attachment will be used. */
- enum anv_subpass_usage * subpass_usage;
+ VkImageLayout first_subpass_layout;
/* The subpass id in which the attachment will be used last. */
uint32_t last_subpass_idx;
struct anv_render_pass {
uint32_t attachment_count;
uint32_t subpass_count;
- VkAttachmentReference * subpass_attachments;
- enum anv_subpass_usage * subpass_usages;
+ /* An array of subpass_count+1 flushes, one per subpass boundary */
+ enum anv_pipe_bits * subpass_flushes;
struct anv_render_pass_attachment * attachments;
struct anv_subpass subpasses[0];
};
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_query_pool, VkQueryPool)
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_render_pass, VkRenderPass)
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_sampler, VkSampler)
+ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_semaphore, VkSemaphore)
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_shader_module, VkShaderModule)
+ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_debug_report_callback, VkDebugReportCallbackEXT)
+ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_ycbcr_conversion, VkSamplerYcbcrConversionKHR)
/* Gen-specific function declarations */
#ifdef genX
# define genX(x) gen9_##x
# include "anv_genX.h"
# undef genX
+# define genX(x) gen10_##x
+# include "anv_genX.h"
+# undef genX
#endif
#endif /* ANV_PRIVATE_H */