#define VG(x)
#endif
+#include "common/gen_clflush.h"
#include "common/gen_device_info.h"
#include "blorp/blorp.h"
-#include "brw_compiler.h"
+#include "compiler/brw_compiler.h"
#include "util/macros.h"
#include "util/list.h"
+#include "util/u_atomic.h"
#include "util/u_vector.h"
-#include "util/vk_alloc.h"
+#include "vk_alloc.h"
/* Pre-declarations needed for WSI entrypoints */
struct wl_surface;
struct anv_buffer;
struct anv_buffer_view;
struct anv_image_view;
+struct anv_instance;
+struct anv_debug_report_callback;
struct gen_l3_config;
#include <vulkan/vk_icd.h>
#include "anv_entrypoints.h"
-#include "brw_context.h"
#include "isl/isl.h"
+#include "common/gen_debug.h"
#include "wsi_common.h"
/* Allowing different clear colors requires us to perform a depth resolve at
*/
#define ANV_HZ_FC_VAL 1.0f
-#define MAX_VBS 31
+#define MAX_VBS 28
#define MAX_SETS 8
#define MAX_RTS 8
#define MAX_VIEWPORTS 16
memcpy((dest), (src), (count) * sizeof(*(src))); \
})
+/* Mapping from anv object to VkDebugReportObjectTypeEXT. New types need
+ * to be added here in order to utilize mapping in debug/error/perf macros.
+ */
+#define REPORT_OBJECT_TYPE(o) \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_instance*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_INSTANCE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_physical_device*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_PHYSICAL_DEVICE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_device*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), const struct anv_device*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_queue*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_QUEUE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_semaphore*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_SEMAPHORE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_cmd_buffer*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_COMMAND_BUFFER_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_fence*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_FENCE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_device_memory*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_MEMORY_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_buffer*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_BUFFER_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_image*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), const struct anv_image*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_event*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_EVENT_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_query_pool*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_QUERY_POOL_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_buffer_view*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_BUFFER_VIEW_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_image_view*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_VIEW_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_shader_module*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_SHADER_MODULE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_pipeline_cache*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_CACHE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_pipeline_layout*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_LAYOUT_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_render_pass*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_RENDER_PASS_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_pipeline*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_set_layout*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_sampler*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_SAMPLER_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_pool*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_POOL_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_set*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_SET_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_framebuffer*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_FRAMEBUFFER_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_cmd_pool*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_COMMAND_POOL_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_surface*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_SURFACE_KHR_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct wsi_swapchain*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_SWAPCHAIN_KHR_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), struct anv_debug_callback*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT_EXT, \
+ __builtin_choose_expr ( \
+ __builtin_types_compatible_p (__typeof (o), void*), \
+ VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT, \
+ /* The void expression results in a compile-time error \
+ when assigning the result to something. */ \
+ (void)0)))))))))))))))))))))))))))))))
+
/* Whenever we generate an error, pass it through this function. Useful for
* debugging, where we can break on it. Only call at error site, not when
* propagating errors. Might be useful to plug in a stack trace here.
*/
-VkResult __vk_errorf(VkResult error, const char *file, int line, const char *format, ...);
+VkResult __vk_errorf(struct anv_instance *instance, const void *object,
+ VkDebugReportObjectTypeEXT type, VkResult error,
+ const char *file, int line, const char *format, ...);
#ifdef DEBUG
-#define vk_error(error) __vk_errorf(error, __FILE__, __LINE__, NULL);
-#define vk_errorf(error, format, ...) __vk_errorf(error, __FILE__, __LINE__, format, ## __VA_ARGS__);
+#define vk_error(error) __vk_errorf(NULL, NULL,\
+ VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT,\
+ error, __FILE__, __LINE__, NULL);
+#define vk_errorf(instance, obj, error, format, ...)\
+ __vk_errorf(instance, obj, REPORT_OBJECT_TYPE(obj), error,\
+ __FILE__, __LINE__, format, ## __VA_ARGS__);
#define anv_debug(format, ...) fprintf(stderr, "debug: " format, ##__VA_ARGS__)
#else
#define vk_error(error) error
void __anv_finishme(const char *file, int line, const char *format, ...)
anv_printflike(3, 4);
+void __anv_perf_warn(struct anv_instance *instance, const void *object,
+ VkDebugReportObjectTypeEXT type, const char *file,
+ int line, const char *format, ...)
+ anv_printflike(6, 7);
void anv_loge(const char *format, ...) anv_printflike(1, 2);
void anv_loge_v(const char *format, va_list va);
+void anv_debug_report(struct anv_instance *instance,
+ VkDebugReportFlagsEXT flags,
+ VkDebugReportObjectTypeEXT object_type,
+ uint64_t handle,
+ size_t location,
+ int32_t messageCode,
+ const char* pLayerPrefix,
+ const char *pMessage);
+
/**
* Print a FINISHME message, including its source location.
*/
} \
} while (0)
+/**
+ * Print a perf warning message. Set INTEL_DEBUG=perf to see these.
+ */
+#define anv_perf_warn(instance, obj, format, ...) \
+ do { \
+ static bool reported = false; \
+ if (!reported && unlikely(INTEL_DEBUG & DEBUG_PERF)) { \
+ __anv_perf_warn(instance, obj, REPORT_OBJECT_TYPE(obj), __FILE__, __LINE__,\
+ format, ##__VA_ARGS__); \
+ reported = true; \
+ } \
+ } while (0)
+
/* A non-fatal assert. Useful for debugging. */
#ifdef DEBUG
#define anv_assert(x) ({ \
#define anv_assert(x)
#endif
-/**
- * If a block of code is annotated with anv_validate, then the block runs only
- * in debug builds.
+/* A multi-pointer allocator
+ *
+ * When copying data structures from the user (such as a render pass), it's
+ * common to need to allocate data for a bunch of different things. Instead
+ * of doing several allocations and having to handle all of the error checking
+ * that entails, it can be easier to do a single allocation. This struct
+ * helps facilitate that. The intended usage looks like this:
+ *
+ * ANV_MULTIALLOC(ma)
+ * anv_multialloc_add(&ma, &main_ptr, 1);
+ * anv_multialloc_add(&ma, &substruct1, substruct1Count);
+ * anv_multialloc_add(&ma, &substruct2, substruct2Count);
+ *
+ * if (!anv_multialloc_alloc(&ma, pAllocator, VK_ALLOCATION_SCOPE_FOO))
+ * return vk_error(VK_ERROR_OUT_OF_HOST_MEORY);
*/
-#ifdef DEBUG
-#define anv_validate if (1)
-#else
-#define anv_validate if (0)
-#endif
+struct anv_multialloc {
+ size_t size;
+ size_t align;
-#define stub_return(v) \
- do { \
- anv_finishme("stub %s", __func__); \
- return (v); \
- } while (0)
+ uint32_t ptr_count;
+ void **ptrs[8];
+};
-#define stub() \
- do { \
- anv_finishme("stub %s", __func__); \
- return; \
- } while (0)
+#define ANV_MULTIALLOC_INIT \
+ ((struct anv_multialloc) { 0, })
-/**
- * A dynamically growable, circular buffer. Elements are added at head and
- * removed from tail. head and tail are free-running uint32_t indices and we
- * only compute the modulo with size when accessing the array. This way,
- * number of bytes in the queue is always head - tail, even in case of
- * wraparound.
- */
+#define ANV_MULTIALLOC(_name) \
+ struct anv_multialloc _name = ANV_MULTIALLOC_INIT
+
+__attribute__((always_inline))
+static inline void
+_anv_multialloc_add(struct anv_multialloc *ma,
+ void **ptr, size_t size, size_t align)
+{
+ size_t offset = align_u64(ma->size, align);
+ ma->size = offset + size;
+ ma->align = MAX2(ma->align, align);
+
+ /* Store the offset in the pointer. */
+ *ptr = (void *)(uintptr_t)offset;
+
+ assert(ma->ptr_count < ARRAY_SIZE(ma->ptrs));
+ ma->ptrs[ma->ptr_count++] = ptr;
+}
+
+#define anv_multialloc_add(_ma, _ptr, _count) \
+ _anv_multialloc_add((_ma), (void **)(_ptr), \
+ (_count) * sizeof(**(_ptr)), __alignof__(**(_ptr)))
+
+__attribute__((always_inline))
+static inline void *
+anv_multialloc_alloc(struct anv_multialloc *ma,
+ const VkAllocationCallbacks *alloc,
+ VkSystemAllocationScope scope)
+{
+ void *ptr = vk_alloc(alloc, ma->size, ma->align, scope);
+ if (!ptr)
+ return NULL;
+
+ /* Fill out each of the pointers with their final value.
+ *
+ * for (uint32_t i = 0; i < ma->ptr_count; i++)
+ * *ma->ptrs[i] = ptr + (uintptr_t)*ma->ptrs[i];
+ *
+ * Unfortunately, even though ma->ptr_count is basically guaranteed to be a
+ * constant, GCC is incapable of figuring this out and unrolling the loop
+ * so we have to give it a little help.
+ */
+ STATIC_ASSERT(ARRAY_SIZE(ma->ptrs) == 8);
+#define _ANV_MULTIALLOC_UPDATE_POINTER(_i) \
+ if ((_i) < ma->ptr_count) \
+ *ma->ptrs[_i] = ptr + (uintptr_t)*ma->ptrs[_i]
+ _ANV_MULTIALLOC_UPDATE_POINTER(0);
+ _ANV_MULTIALLOC_UPDATE_POINTER(1);
+ _ANV_MULTIALLOC_UPDATE_POINTER(2);
+ _ANV_MULTIALLOC_UPDATE_POINTER(3);
+ _ANV_MULTIALLOC_UPDATE_POINTER(4);
+ _ANV_MULTIALLOC_UPDATE_POINTER(5);
+ _ANV_MULTIALLOC_UPDATE_POINTER(6);
+ _ANV_MULTIALLOC_UPDATE_POINTER(7);
+#undef _ANV_MULTIALLOC_UPDATE_POINTER
+
+ return ptr;
+}
+
+__attribute__((always_inline))
+static inline void *
+anv_multialloc_alloc2(struct anv_multialloc *ma,
+ const VkAllocationCallbacks *parent_alloc,
+ const VkAllocationCallbacks *alloc,
+ VkSystemAllocationScope scope)
+{
+ return anv_multialloc_alloc(ma, alloc ? alloc : parent_alloc, scope);
+}
struct anv_bo {
uint32_t gem_handle;
uint64_t size;
void *map;
- /* We need to set the WRITE flag on winsys bos so GEM will know we're
- * writing to them and synchronize uses on other rings (eg if the display
- * server uses the blitter ring).
- */
- bool is_winsys_bo;
+ /** Flags to pass to the kernel through drm_i915_exec_object2::flags */
+ uint32_t flags;
};
static inline void
bo->offset = -1;
bo->size = size;
bo->map = NULL;
- bo->is_winsys_bo = false;
+ bo->flags = 0;
}
/* Represents a lock-free linked list of "free" things. This is used by
*/
struct u_vector mmap_cleanups;
- uint32_t block_size;
-
- union anv_free_list free_list;
struct anv_block_state state;
- union anv_free_list back_free_list;
struct anv_block_state back_state;
};
-/* Block pools are backed by a fixed-size 2GB memfd */
-#define BLOCK_POOL_MEMFD_SIZE (1ull << 32)
+/* Block pools are backed by a fixed-size 1GB memfd */
+#define BLOCK_POOL_MEMFD_SIZE (1ul << 30)
/* The center of the block pool is also the middle of the memfd. This may
* change in the future if we decide differently for some reason.
void *map;
};
+#define ANV_STATE_NULL ((struct anv_state) { .alloc_size = 0 })
+
struct anv_fixed_size_state_pool {
- size_t state_size;
union anv_free_list free_list;
struct anv_block_state block;
};
#define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2 + 1)
struct anv_state_pool {
- struct anv_block_pool *block_pool;
+ struct anv_block_pool block_pool;
+
+ /* The size of blocks which will be allocated from the block pool */
+ uint32_t block_size;
+
+ /** Free list for "back" allocations */
+ union anv_free_list back_alloc_free_list;
+
struct anv_fixed_size_state_pool buckets[ANV_STATE_BUCKETS];
};
struct anv_state_stream_block;
struct anv_state_stream {
- struct anv_block_pool *block_pool;
-
- /* The current working block */
- struct anv_state_stream_block *block;
-
- /* Offset at which the current block starts */
- uint32_t start;
- /* Offset at which to allocate the next state */
- uint32_t next;
- /* Offset at which the current block ends */
- uint32_t end;
-};
-
-#define CACHELINE_SIZE 64
-#define CACHELINE_MASK 63
+ struct anv_state_pool *state_pool;
-static inline void
-anv_clflush_range(void *start, size_t size)
-{
- void *p = (void *) (((uintptr_t) start) & ~CACHELINE_MASK);
- void *end = start + size;
+ /* The size of blocks to allocate from the state pool */
+ uint32_t block_size;
- while (p < end) {
- __builtin_ia32_clflush(p);
- p += CACHELINE_SIZE;
- }
-}
+ /* Current block we're allocating from */
+ struct anv_state block;
-static inline void
-anv_flush_range(void *start, size_t size)
-{
- __builtin_ia32_mfence();
- anv_clflush_range(start, size);
-}
+ /* Offset into the current block at which to allocate the next state */
+ uint32_t next;
-static inline void
-anv_invalidate_range(void *start, size_t size)
-{
- anv_clflush_range(start, size);
- __builtin_ia32_mfence();
-}
+ /* List of all blocks allocated from this pool */
+ struct anv_state_stream_block *block_list;
+};
+/* The block_pool functions exported for testing only. The block pool should
+ * only be used via a state pool (see below).
+ */
VkResult anv_block_pool_init(struct anv_block_pool *pool,
- struct anv_device *device, uint32_t block_size);
+ struct anv_device *device,
+ uint32_t initial_size);
void anv_block_pool_finish(struct anv_block_pool *pool);
-int32_t anv_block_pool_alloc(struct anv_block_pool *pool);
-int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool);
-void anv_block_pool_free(struct anv_block_pool *pool, int32_t offset);
-void anv_state_pool_init(struct anv_state_pool *pool,
- struct anv_block_pool *block_pool);
+int32_t anv_block_pool_alloc(struct anv_block_pool *pool,
+ uint32_t block_size);
+int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool,
+ uint32_t block_size);
+
+VkResult anv_state_pool_init(struct anv_state_pool *pool,
+ struct anv_device *device,
+ uint32_t block_size);
void anv_state_pool_finish(struct anv_state_pool *pool);
struct anv_state anv_state_pool_alloc(struct anv_state_pool *pool,
- size_t state_size, size_t alignment);
+ uint32_t state_size, uint32_t alignment);
+struct anv_state anv_state_pool_alloc_back(struct anv_state_pool *pool);
void anv_state_pool_free(struct anv_state_pool *pool, struct anv_state state);
void anv_state_stream_init(struct anv_state_stream *stream,
- struct anv_block_pool *block_pool);
+ struct anv_state_pool *state_pool,
+ uint32_t block_size);
void anv_state_stream_finish(struct anv_state_stream *stream);
struct anv_state anv_state_stream_alloc(struct anv_state_stream *stream,
uint32_t size, uint32_t alignment);
gl_shader_stage stage,
unsigned per_thread_scratch);
+/** Implements a BO cache that ensures a 1-1 mapping of GEM BOs to anv_bos */
+struct anv_bo_cache {
+ struct hash_table *bo_map;
+ pthread_mutex_t mutex;
+};
+
+VkResult anv_bo_cache_init(struct anv_bo_cache *cache);
+void anv_bo_cache_finish(struct anv_bo_cache *cache);
+VkResult anv_bo_cache_alloc(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ uint64_t size, struct anv_bo **bo);
+VkResult anv_bo_cache_import(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ int fd, uint64_t size, struct anv_bo **bo);
+VkResult anv_bo_cache_export(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ struct anv_bo *bo_in, int *fd_out);
+void anv_bo_cache_release(struct anv_device *device,
+ struct anv_bo_cache *cache,
+ struct anv_bo *bo);
+
+struct anv_memory_type {
+ /* Standard bits passed on to the client */
+ VkMemoryPropertyFlags propertyFlags;
+ uint32_t heapIndex;
+
+ /* Driver-internal book-keeping */
+ VkBufferUsageFlags valid_buffer_usage;
+};
+
+struct anv_memory_heap {
+ /* Standard bits passed on to the client */
+ VkDeviceSize size;
+ VkMemoryHeapFlags flags;
+
+ /* Driver-internal book-keeping */
+ bool supports_48bit_addresses;
+};
+
struct anv_physical_device {
VK_LOADER_DATA _loader_data;
char path[20];
const char * name;
struct gen_device_info info;
- uint64_t aperture_size;
+ /** Amount of "GPU memory" we want to advertise
+ *
+ * Clearly, this value is bogus since Intel is a UMA architecture. On
+ * gen7 platforms, we are limited by GTT size unless we want to implement
+ * fine-grained tracking and GTT splitting. On Broadwell and above we are
+ * practically unlimited. However, we will never report more than 3/4 of
+ * the total system ram to try and avoid running out of RAM.
+ */
+ bool supports_48bit_addresses;
struct brw_compiler * compiler;
struct isl_device isl_dev;
int cmd_parser_version;
+ bool has_exec_async;
+ bool has_exec_fence;
+ bool has_syncobj;
+ bool has_syncobj_wait;
uint32_t eu_total;
uint32_t subslice_total;
- uint8_t uuid[VK_UUID_SIZE];
+ struct {
+ uint32_t type_count;
+ struct anv_memory_type types[VK_MAX_MEMORY_TYPES];
+ uint32_t heap_count;
+ struct anv_memory_heap heaps[VK_MAX_MEMORY_HEAPS];
+ } memory;
+
+ uint8_t pipeline_cache_uuid[VK_UUID_SIZE];
+ uint8_t driver_uuid[VK_UUID_SIZE];
+ uint8_t device_uuid[VK_UUID_SIZE];
struct wsi_device wsi_device;
int local_fd;
};
+struct anv_debug_report_callback {
+ /* Link in the 'callbacks' list in anv_instance struct. */
+ struct list_head link;
+ VkDebugReportFlagsEXT flags;
+ PFN_vkDebugReportCallbackEXT callback;
+ void * data;
+};
+
struct anv_instance {
VK_LOADER_DATA _loader_data;
uint32_t apiVersion;
int physicalDeviceCount;
struct anv_physical_device physicalDevice;
+
+ /* VK_EXT_debug_report debug callbacks */
+ pthread_mutex_t callbacks_mutex;
+ struct list_head callbacks;
+ struct anv_debug_report_callback destroy_debug_cb;
};
VkResult anv_init_wsi(struct anv_physical_device *physical_device);
void anv_finish_wsi(struct anv_physical_device *physical_device);
+bool anv_instance_extension_supported(const char *name);
+uint32_t anv_physical_device_api_version(struct anv_physical_device *dev);
+bool anv_physical_device_extension_supported(struct anv_physical_device *dev,
+ const char *name);
+
struct anv_queue {
VK_LOADER_DATA _loader_data;
struct anv_bo_pool batch_bo_pool;
- struct anv_block_pool dynamic_state_block_pool;
- struct anv_state_pool dynamic_state_pool;
+ struct anv_bo_cache bo_cache;
- struct anv_block_pool instruction_block_pool;
+ struct anv_state_pool dynamic_state_pool;
struct anv_state_pool instruction_state_pool;
-
- struct anv_block_pool surface_state_block_pool;
struct anv_state_pool surface_state_pool;
struct anv_bo workaround_bo;
+ struct anv_bo trivial_batch_bo;
struct anv_pipeline_cache blorp_shader_cache;
struct blorp_context blorp;
pthread_mutex_t mutex;
pthread_cond_t queue_submit;
+ bool lost;
};
static void inline
if (device->info.has_llc)
return;
- anv_flush_range(state.map, state.alloc_size);
+ gen_flush_range(state.map, state.alloc_size);
}
void anv_device_init_blorp(struct anv_device *device);
VkResult anv_device_execbuf(struct anv_device *device,
struct drm_i915_gem_execbuffer2 *execbuf,
struct anv_bo **execbuf_bos);
+VkResult anv_device_query_status(struct anv_device *device);
+VkResult anv_device_bo_busy(struct anv_device *device, struct anv_bo *bo);
+VkResult anv_device_wait(struct anv_device *device, struct anv_bo *bo,
+ int64_t timeout);
void* anv_gem_mmap(struct anv_device *device,
uint32_t gem_handle, uint64_t offset, uint64_t size, uint32_t flags);
void anv_gem_munmap(void *p, uint64_t size);
-uint32_t anv_gem_create(struct anv_device *device, size_t size);
+uint32_t anv_gem_create(struct anv_device *device, uint64_t size);
void anv_gem_close(struct anv_device *device, uint32_t gem_handle);
uint32_t anv_gem_userptr(struct anv_device *device, void *mem, size_t size);
+int anv_gem_busy(struct anv_device *device, uint32_t gem_handle);
int anv_gem_wait(struct anv_device *device, uint32_t gem_handle, int64_t *timeout_ns);
int anv_gem_execbuffer(struct anv_device *device,
struct drm_i915_gem_execbuffer2 *execbuf);
uint32_t stride, uint32_t tiling);
int anv_gem_create_context(struct anv_device *device);
int anv_gem_destroy_context(struct anv_device *device, int context);
+int anv_gem_get_context_param(int fd, int context, uint32_t param,
+ uint64_t *value);
int anv_gem_get_param(int fd, uint32_t param);
bool anv_gem_get_bit6_swizzle(int fd, uint32_t tiling);
int anv_gem_get_aperture(int fd, uint64_t *size);
+bool anv_gem_supports_48b_addresses(int fd);
+int anv_gem_gpu_get_reset_stats(struct anv_device *device,
+ uint32_t *active, uint32_t *pending);
int anv_gem_handle_to_fd(struct anv_device *device, uint32_t gem_handle);
uint32_t anv_gem_fd_to_handle(struct anv_device *device, int fd);
int anv_gem_set_caching(struct anv_device *device, uint32_t gem_handle, uint32_t caching);
int anv_gem_set_domain(struct anv_device *device, uint32_t gem_handle,
uint32_t read_domains, uint32_t write_domain);
+int anv_gem_sync_file_merge(struct anv_device *device, int fd1, int fd2);
+uint32_t anv_gem_syncobj_create(struct anv_device *device, uint32_t flags);
+void anv_gem_syncobj_destroy(struct anv_device *device, uint32_t handle);
+int anv_gem_syncobj_handle_to_fd(struct anv_device *device, uint32_t handle);
+uint32_t anv_gem_syncobj_fd_to_handle(struct anv_device *device, int fd);
+int anv_gem_syncobj_export_sync_file(struct anv_device *device,
+ uint32_t handle);
+int anv_gem_syncobj_import_sync_file(struct anv_device *device,
+ uint32_t handle, int fd);
+void anv_gem_syncobj_reset(struct anv_device *device, uint32_t handle);
+bool anv_gem_supports_syncobj_wait(int fd);
+int anv_gem_syncobj_wait(struct anv_device *device,
+ uint32_t *handles, uint32_t num_handles,
+ int64_t abs_timeout_ns, bool wait_all);
VkResult anv_bo_init_new(struct anv_bo *bo, struct anv_device *device, uint64_t size);
struct anv_reloc_list {
- size_t num_relocs;
- size_t array_length;
+ uint32_t num_relocs;
+ uint32_t array_length;
struct drm_i915_gem_relocation_entry * relocs;
struct anv_bo ** reloc_bos;
};
void anv_reloc_list_finish(struct anv_reloc_list *list,
const VkAllocationCallbacks *alloc);
-uint64_t anv_reloc_list_add(struct anv_reloc_list *list,
+VkResult anv_reloc_list_add(struct anv_reloc_list *list,
const VkAllocationCallbacks *alloc,
uint32_t offset, struct anv_bo *target_bo,
uint32_t delta);
struct anv_bo bo;
/* Bytes actually consumed in this batch BO */
- size_t length;
+ uint32_t length;
struct anv_reloc_list relocs;
};
*/
VkResult (*extend_cb)(struct anv_batch *, void *);
void * user_data;
+
+ /**
+ * Current error status of the command buffer. Used to track inconsistent
+ * or incomplete command buffer states that are the consequence of run-time
+ * errors such as out of memory scenarios. We want to track this in the
+ * batch because the command buffer object is not visible to some parts
+ * of the driver.
+ */
+ VkResult status;
};
void *anv_batch_emit_dwords(struct anv_batch *batch, int num_dwords);
VkResult anv_device_submit_simple_batch(struct anv_device *device,
struct anv_batch *batch);
+static inline VkResult
+anv_batch_set_error(struct anv_batch *batch, VkResult error)
+{
+ assert(error != VK_SUCCESS);
+ if (batch->status == VK_SUCCESS)
+ batch->status = error;
+ return batch->status;
+}
+
+static inline bool
+anv_batch_has_error(struct anv_batch *batch)
+{
+ return batch->status != VK_SUCCESS;
+}
+
struct anv_address {
struct anv_bo *bo;
uint32_t offset;
VG(VALGRIND_CHECK_MEM_IS_DEFINED(dst, __anv_cmd_length(struc) * 4)); \
} while (0)
-#define anv_batch_emitn(batch, n, cmd, ...) ({ \
- void *__dst = anv_batch_emit_dwords(batch, n); \
- struct cmd __template = { \
- __anv_cmd_header(cmd), \
- .DWordLength = n - __anv_cmd_length_bias(cmd), \
- __VA_ARGS__ \
- }; \
- __anv_cmd_pack(cmd)(batch, __dst, &__template); \
- __dst; \
+#define anv_batch_emitn(batch, n, cmd, ...) ({ \
+ void *__dst = anv_batch_emit_dwords(batch, n); \
+ if (__dst) { \
+ struct cmd __template = { \
+ __anv_cmd_header(cmd), \
+ .DWordLength = n - __anv_cmd_length_bias(cmd), \
+ __VA_ARGS__ \
+ }; \
+ __anv_cmd_pack(cmd)(batch, __dst, &__template); \
+ } \
+ __dst; \
})
#define anv_batch_emit_merge(batch, dwords0, dwords1) \
\
STATIC_ASSERT(ARRAY_SIZE(dwords0) == ARRAY_SIZE(dwords1)); \
dw = anv_batch_emit_dwords((batch), ARRAY_SIZE(dwords0)); \
+ if (!dw) \
+ break; \
for (uint32_t i = 0; i < ARRAY_SIZE(dwords0); i++) \
dw[i] = (dwords0)[i] | (dwords1)[i]; \
VG(VALGRIND_CHECK_MEM_IS_DEFINED(dw, ARRAY_SIZE(dwords0) * 4));\
.IndextoMOCSTables = 1 \
}
+/* Cannonlake MOCS defines are duplicates of Skylake MOCS defines. */
+#define GEN10_MOCS (struct GEN10_MEMORY_OBJECT_CONTROL_STATE) { \
+ /* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */ \
+ .IndextoMOCSTables = 2 \
+ }
+
+#define GEN10_MOCS_PTE { \
+ /* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */ \
+ .IndextoMOCSTables = 1 \
+ }
+
struct anv_device_memory {
- struct anv_bo bo;
- uint32_t type_index;
+ struct anv_bo * bo;
+ struct anv_memory_type * type;
VkDeviceSize map_size;
void * map;
};
union {
struct {
+ VkImageLayout layout;
struct anv_image_view *image_view;
struct anv_sampler *sampler;
};
+ struct {
+ struct anv_buffer *buffer;
+ uint64_t offset;
+ uint64_t range;
+ };
+
struct anv_buffer_view *buffer_view;
};
};
void
anv_descriptor_set_write_image_view(struct anv_descriptor_set *set,
+ const struct gen_device_info * const devinfo,
+ const VkDescriptorImageInfo * const info,
VkDescriptorType type,
- VkImageView _image_view,
- VkSampler _sampler,
uint32_t binding,
uint32_t element);
uint8_t set;
/* Binding in the descriptor set */
- uint8_t binding;
+ uint32_t binding;
/* Index in the binding */
- uint8_t index;
+ uint32_t index;
/* Input attachment index (relative to the subpass) */
uint8_t input_attachment_index;
VkDeviceSize offset;
};
+static inline uint64_t
+anv_buffer_get_range(struct anv_buffer *buffer, uint64_t offset, uint64_t range)
+{
+ assert(offset <= buffer->size);
+ if (range == VK_WHOLE_SIZE) {
+ return buffer->size - offset;
+ } else {
+ assert(range <= buffer->size);
+ return range;
+ }
+}
+
enum anv_cmd_dirty_bits {
ANV_CMD_DIRTY_DYNAMIC_VIEWPORT = 1 << 0, /* VK_DYNAMIC_STATE_VIEWPORT */
ANV_CMD_DIRTY_DYNAMIC_SCISSOR = 1 << 1, /* VK_DYNAMIC_STATE_SCISSOR */
ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT | \
ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT)
+static inline enum anv_pipe_bits
+anv_pipe_flush_bits_for_access_flags(VkAccessFlags flags)
+{
+ enum anv_pipe_bits pipe_bits = 0;
+
+ unsigned b;
+ for_each_bit(b, flags) {
+ switch ((VkAccessFlagBits)(1 << b)) {
+ case VK_ACCESS_SHADER_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_DATA_CACHE_FLUSH_BIT;
+ break;
+ case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
+ break;
+ case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
+ break;
+ case VK_ACCESS_TRANSFER_WRITE_BIT:
+ pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
+ pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
+ break;
+ default:
+ break; /* Nothing to do */
+ }
+ }
+
+ return pipe_bits;
+}
+
+static inline enum anv_pipe_bits
+anv_pipe_invalidate_bits_for_access_flags(VkAccessFlags flags)
+{
+ enum anv_pipe_bits pipe_bits = 0;
+
+ unsigned b;
+ for_each_bit(b, flags) {
+ switch ((VkAccessFlagBits)(1 << b)) {
+ case VK_ACCESS_INDIRECT_COMMAND_READ_BIT:
+ case VK_ACCESS_INDEX_READ_BIT:
+ case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT:
+ pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
+ break;
+ case VK_ACCESS_UNIFORM_READ_BIT:
+ pipe_bits |= ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
+ pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
+ break;
+ case VK_ACCESS_SHADER_READ_BIT:
+ case VK_ACCESS_INPUT_ATTACHMENT_READ_BIT:
+ case VK_ACCESS_TRANSFER_READ_BIT:
+ pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
+ break;
+ default:
+ break; /* Nothing to do */
+ }
+ }
+
+ return pipe_bits;
+}
+
struct anv_vertex_binding {
struct anv_buffer * buffer;
VkDeviceSize offset;
uint32_t base_vertex;
uint32_t base_instance;
- /* Offsets and ranges for dynamically bound buffers */
- struct {
- uint32_t offset;
- uint32_t range;
- } dynamic[MAX_DYNAMIC_BUFFERS];
-
/* Image data for image_load_store on pre-SKL */
struct brw_image_param images[MAX_IMAGES];
};
bool fast_clear;
VkClearValue clear_value;
bool clear_color_is_zero_one;
+ bool clear_color_is_zero;
};
/** State required while building cmd buffer */
uint32_t restart_index;
struct anv_vertex_binding vertex_bindings[MAX_VBS];
struct anv_descriptor_set * descriptors[MAX_SETS];
+ uint32_t dynamic_offsets[MAX_DYNAMIC_BUFFERS];
VkShaderStageFlags push_constant_stages;
struct anv_push_constants * push_constants[MESA_SHADER_STAGES];
struct anv_state binding_tables[MESA_SHADER_STAGES];
*
* initialized by anv_cmd_buffer_init_batch_bo_chain()
*/
- struct u_vector bt_blocks;
+ struct u_vector bt_block_states;
uint32_t bt_next;
struct anv_reloc_list surface_relocs;
struct anv_cmd_buffer *secondary);
void anv_cmd_buffer_prepare_execbuf(struct anv_cmd_buffer *cmd_buffer);
VkResult anv_cmd_buffer_execbuf(struct anv_device *device,
- struct anv_cmd_buffer *cmd_buffer);
+ struct anv_cmd_buffer *cmd_buffer,
+ const VkSemaphore *in_semaphores,
+ uint32_t num_in_semaphores,
+ const VkSemaphore *out_semaphores,
+ uint32_t num_out_semaphores,
+ VkFence fence);
VkResult anv_cmd_buffer_reset(struct anv_cmd_buffer *cmd_buffer);
const struct anv_image_view *
anv_cmd_buffer_get_depth_stencil_view(const struct anv_cmd_buffer *cmd_buffer);
-struct anv_state
+VkResult
anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
uint32_t num_entries,
- uint32_t *state_offset);
+ uint32_t *state_offset,
+ struct anv_state *bt_state);
void anv_cmd_buffer_dump(struct anv_cmd_buffer *cmd_buffer);
-enum anv_fence_state {
+enum anv_fence_type {
+ ANV_FENCE_TYPE_NONE = 0,
+ ANV_FENCE_TYPE_BO,
+ ANV_FENCE_TYPE_SYNCOBJ,
+};
+
+enum anv_bo_fence_state {
/** Indicates that this is a new (or newly reset fence) */
- ANV_FENCE_STATE_RESET,
+ ANV_BO_FENCE_STATE_RESET,
/** Indicates that this fence has been submitted to the GPU but is still
* (as far as we know) in use by the GPU.
*/
- ANV_FENCE_STATE_SUBMITTED,
+ ANV_BO_FENCE_STATE_SUBMITTED,
+
+ ANV_BO_FENCE_STATE_SIGNALED,
+};
+
+struct anv_fence_impl {
+ enum anv_fence_type type;
+
+ union {
+ /** Fence implementation for BO fences
+ *
+ * These fences use a BO and a set of CPU-tracked state flags. The BO
+ * is added to the object list of the last execbuf call in a QueueSubmit
+ * and is marked EXEC_WRITE. The state flags track when the BO has been
+ * submitted to the kernel. We need to do this because Vulkan lets you
+ * wait on a fence that has not yet been submitted and I915_GEM_BUSY
+ * will say it's idle in this case.
+ */
+ struct {
+ struct anv_bo bo;
+ enum anv_bo_fence_state state;
+ } bo;
- ANV_FENCE_STATE_SIGNALED,
+ /** DRM syncobj handle for syncobj-based fences */
+ uint32_t syncobj;
+ };
};
struct anv_fence {
- struct anv_bo bo;
- struct drm_i915_gem_execbuffer2 execbuf;
- struct drm_i915_gem_exec_object2 exec2_objects[1];
- enum anv_fence_state state;
+ /* Permanent fence state. Every fence has some form of permanent state
+ * (type != ANV_SEMAPHORE_TYPE_NONE). This may be a BO to fence on (for
+ * cross-process fences0 or it could just be a dummy for use internally.
+ */
+ struct anv_fence_impl permanent;
+
+ /* Temporary fence state. A fence *may* have temporary state. That state
+ * is added to the fence by an import operation and is reset back to
+ * ANV_SEMAPHORE_TYPE_NONE when the fence is reset. A fence with temporary
+ * state cannot be signaled because the fence must already be signaled
+ * before the temporary state can be exported from the fence in the other
+ * process and imported here.
+ */
+ struct anv_fence_impl temporary;
};
struct anv_event {
struct anv_state state;
};
+enum anv_semaphore_type {
+ ANV_SEMAPHORE_TYPE_NONE = 0,
+ ANV_SEMAPHORE_TYPE_DUMMY,
+ ANV_SEMAPHORE_TYPE_BO,
+ ANV_SEMAPHORE_TYPE_SYNC_FILE,
+ ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ,
+};
+
+struct anv_semaphore_impl {
+ enum anv_semaphore_type type;
+
+ union {
+ /* A BO representing this semaphore when type == ANV_SEMAPHORE_TYPE_BO.
+ * This BO will be added to the object list on any execbuf2 calls for
+ * which this semaphore is used as a wait or signal fence. When used as
+ * a signal fence, the EXEC_OBJECT_WRITE flag will be set.
+ */
+ struct anv_bo *bo;
+
+ /* The sync file descriptor when type == AKV_SEMAPHORE_TYPE_SYNC_FILE.
+ * If the semaphore is in the unsignaled state due to either just being
+ * created or because it has been used for a wait, fd will be -1.
+ */
+ int fd;
+
+ /* Sync object handle when type == AKV_SEMAPHORE_TYPE_DRM_SYNCOBJ.
+ * Unlike GEM BOs, DRM sync objects aren't deduplicated by the kernel on
+ * import so we don't need to bother with a userspace cache.
+ */
+ uint32_t syncobj;
+ };
+};
+
+struct anv_semaphore {
+ /* Permanent semaphore state. Every semaphore has some form of permanent
+ * state (type != ANV_SEMAPHORE_TYPE_NONE). This may be a BO to fence on
+ * (for cross-process semaphores0 or it could just be a dummy for use
+ * internally.
+ */
+ struct anv_semaphore_impl permanent;
+
+ /* Temporary semaphore state. A semaphore *may* have temporary state.
+ * That state is added to the semaphore by an import operation and is reset
+ * back to ANV_SEMAPHORE_TYPE_NONE when the semaphore is waited on. A
+ * semaphore with temporary state cannot be signaled because the semaphore
+ * must already be signaled before the temporary state can be exported from
+ * the semaphore in the other process and imported here.
+ */
+ struct anv_semaphore_impl temporary;
+};
+
+void anv_semaphore_reset_temporary(struct anv_device *device,
+ struct anv_semaphore *semaphore);
+
struct anv_shader_module {
unsigned char sha1[20];
uint32_t size;
char data[0];
};
-void anv_hash_shader(unsigned char *hash, const void *key, size_t key_size,
- struct anv_shader_module *module,
- const char *entrypoint,
- const struct anv_pipeline_layout *pipeline_layout,
- const VkSpecializationInfo *spec_info);
-
static inline gl_shader_stage
vk_to_mesa_shader_stage(VkShaderStageFlagBits vk_stage)
{
static inline void
anv_shader_bin_ref(struct anv_shader_bin *shader)
{
- assert(shader->ref_cnt >= 1);
- __sync_fetch_and_add(&shader->ref_cnt, 1);
+ assert(shader && shader->ref_cnt >= 1);
+ p_atomic_inc(&shader->ref_cnt);
}
static inline void
anv_shader_bin_unref(struct anv_device *device, struct anv_shader_bin *shader)
{
- assert(shader->ref_cnt >= 1);
- if (__sync_fetch_and_add(&shader->ref_cnt, -1) == 1)
+ assert(shader && shader->ref_cnt >= 1);
+ if (p_atomic_dec_zero(&shader->ref_cnt))
anv_shader_bin_destroy(device, shader);
}
uint32_t dynamic_state_mask;
struct anv_dynamic_state dynamic_state;
+ struct anv_subpass * subpass;
struct anv_pipeline_layout * layout;
bool needs_data_cache;
bool writes_stencil;
bool stencil_test_enable;
bool depth_clamp_enable;
+ bool sample_shading_enable;
bool kill_pixel;
struct {
struct anv_surface aux_surface;
};
+/* Returns the number of auxiliary buffer levels attached to an image. */
+static inline uint8_t
+anv_image_aux_levels(const struct anv_image * const image)
+{
+ assert(image);
+ return image->aux_surface.isl.size > 0 ? image->aux_surface.isl.levels : 0;
+}
+
+/* Returns the number of auxiliary buffer layers attached to an image. */
+static inline uint32_t
+anv_image_aux_layers(const struct anv_image * const image,
+ const uint8_t miplevel)
+{
+ assert(image);
+
+ /* The miplevel must exist in the main buffer. */
+ assert(miplevel < image->levels);
+
+ if (miplevel >= anv_image_aux_levels(image)) {
+ /* There are no layers with auxiliary data because the miplevel has no
+ * auxiliary data.
+ */
+ return 0;
+ } else {
+ return MAX2(image->aux_surface.isl.logical_level0_px.array_len,
+ image->aux_surface.isl.logical_level0_px.depth >> miplevel);
+ }
+}
+
+static inline unsigned
+anv_fast_clear_state_entry_size(const struct anv_device *device)
+{
+ assert(device);
+ /* Entry contents:
+ * +--------------------------------------------+
+ * | clear value dword(s) | needs resolve dword |
+ * +--------------------------------------------+
+ */
+
+ /* Ensure that the needs resolve dword is in fact dword-aligned to enable
+ * GPU memcpy operations.
+ */
+ assert(device->isl_dev.ss.clear_value_size % 4 == 0);
+ return device->isl_dev.ss.clear_value_size + 4;
+}
+
/* Returns true if a HiZ-enabled depth buffer can be sampled from. */
static inline bool
-anv_can_sample_with_hiz(uint8_t gen, uint32_t samples)
+anv_can_sample_with_hiz(const struct gen_device_info * const devinfo,
+ const VkImageAspectFlags aspect_mask,
+ const uint32_t samples)
{
- return gen >= 8 && samples == 1;
+ /* Validate the inputs. */
+ assert(devinfo && aspect_mask && samples);
+ return devinfo->gen >= 8 && (aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) &&
+ samples == 1;
}
void
anv_gen8_hiz_op_resolve(struct anv_cmd_buffer *cmd_buffer,
const struct anv_image *image,
enum blorp_hiz_op op);
+void
+anv_ccs_resolve(struct anv_cmd_buffer * const cmd_buffer,
+ const struct anv_state surface_state,
+ const struct anv_image * const image,
+ const uint8_t level, const uint32_t layer_count,
+ const enum blorp_fast_clear_op op);
+
+void
+anv_image_fast_clear(struct anv_cmd_buffer *cmd_buffer,
+ const struct anv_image *image,
+ const uint32_t base_level, const uint32_t level_count,
+ const uint32_t base_layer, uint32_t layer_count);
enum isl_aux_usage
anv_layout_to_aux_usage(const struct gen_device_info * const devinfo,
const struct anv_image *image,
const VkImageAspectFlags aspects,
const VkImageLayout layout);
-static inline uint32_t
-anv_get_layerCount(const struct anv_image *image,
- const VkImageSubresourceRange *range)
-{
- return range->layerCount == VK_REMAINING_ARRAY_LAYERS ?
- image->array_size - range->baseArrayLayer : range->layerCount;
-}
+
+/* This is defined as a macro so that it works for both
+ * VkImageSubresourceRange and VkImageSubresourceLayers
+ */
+#define anv_get_layerCount(_image, _range) \
+ ((_range)->layerCount == VK_REMAINING_ARRAY_LAYERS ? \
+ (_image)->array_size - (_range)->baseArrayLayer : (_range)->layerCount)
static inline uint32_t
anv_get_levelCount(const struct anv_image *image,
VkFormat vk_format;
VkExtent3D extent; /**< Extent of VkImageViewCreateInfo::baseMipLevel. */
- /** RENDER_SURFACE_STATE when using image as a sampler surface. */
- struct anv_state sampler_surface_state;
+ /**
+ * RENDER_SURFACE_STATE when using image as a sampler surface with an image
+ * layout of SHADER_READ_ONLY_OPTIMAL or DEPTH_STENCIL_READ_ONLY_OPTIMAL.
+ */
+ enum isl_aux_usage optimal_sampler_aux_usage;
+ struct anv_state optimal_sampler_surface_state;
+
+ /**
+ * RENDER_SURFACE_STATE when using image as a sampler surface with an image
+ * layout of GENERAL.
+ */
+ enum isl_aux_usage general_sampler_aux_usage;
+ struct anv_state general_sampler_surface_state;
/**
* RENDER_SURFACE_STATE when using image as a storage image. Separate states
uint32_t offset, uint32_t range,
uint32_t stride);
-void anv_image_view_fill_image_param(struct anv_device *device,
- struct anv_image_view *view,
- struct brw_image_param *param);
-void anv_buffer_view_fill_image_param(struct anv_device *device,
- struct anv_buffer_view *view,
- struct brw_image_param *param);
-
struct anv_sampler {
uint32_t state[4];
};
};
struct anv_subpass {
+ uint32_t attachment_count;
+
+ /**
+ * A pointer to all attachment references used in this subpass.
+ * Only valid if ::attachment_count > 0.
+ */
+ VkAttachmentReference * attachments;
uint32_t input_count;
- uint32_t * input_attachments;
+ VkAttachmentReference * input_attachments;
uint32_t color_count;
- uint32_t * color_attachments;
- uint32_t * resolve_attachments;
+ VkAttachmentReference * color_attachments;
+ VkAttachmentReference * resolve_attachments;
- /* TODO: Consider storing the depth/stencil VkAttachmentReference
- * instead of its two structure members (below) individually.
- */
- uint32_t depth_stencil_attachment;
- VkImageLayout depth_stencil_layout;
+ VkAttachmentReference depth_stencil_attachment;
+
+ uint32_t view_mask;
/** Subpass has a depth/stencil self-dependency */
bool has_ds_self_dep;
bool has_resolve;
};
-enum anv_subpass_usage {
- ANV_SUBPASS_USAGE_DRAW = (1 << 0),
- ANV_SUBPASS_USAGE_INPUT = (1 << 1),
- ANV_SUBPASS_USAGE_RESOLVE_SRC = (1 << 2),
- ANV_SUBPASS_USAGE_RESOLVE_DST = (1 << 3),
-};
+static inline unsigned
+anv_subpass_view_count(const struct anv_subpass *subpass)
+{
+ return MAX2(1, _mesa_bitcount(subpass->view_mask));
+}
struct anv_render_pass_attachment {
/* TODO: Consider using VkAttachmentDescription instead of storing each of
VkAttachmentLoadOp stencil_load_op;
VkImageLayout initial_layout;
VkImageLayout final_layout;
-
- /* An array, indexed by subpass id, of how the attachment will be used. */
- enum anv_subpass_usage * subpass_usage;
+ VkImageLayout first_subpass_layout;
/* The subpass id in which the attachment will be used last. */
uint32_t last_subpass_idx;
struct anv_render_pass {
uint32_t attachment_count;
uint32_t subpass_count;
- uint32_t * subpass_attachments;
- enum anv_subpass_usage * subpass_usages;
+ /* An array of subpass_count+1 flushes, one per subpass boundary */
+ enum anv_pipe_bits * subpass_flushes;
struct anv_render_pass_attachment * attachments;
struct anv_subpass subpasses[0];
};
-struct anv_query_pool_slot {
- uint64_t begin;
- uint64_t end;
- uint64_t available;
-};
+#define ANV_PIPELINE_STATISTICS_MASK 0x000007ff
struct anv_query_pool {
VkQueryType type;
+ VkQueryPipelineStatisticFlags pipeline_statistics;
+ /** Stride between slots, in bytes */
+ uint32_t stride;
+ /** Number of slots in this query pool */
uint32_t slots;
struct anv_bo bo;
};
void anv_dump_add_framebuffer(struct anv_cmd_buffer *cmd_buffer,
struct anv_framebuffer *fb);
+static inline uint32_t
+anv_get_subpass_id(const struct anv_cmd_state * const cmd_state)
+{
+ /* This function must be called from within a subpass. */
+ assert(cmd_state->pass && cmd_state->subpass);
+
+ const uint32_t subpass_id = cmd_state->subpass - cmd_state->pass->subpasses;
+
+ /* The id of this subpass shouldn't exceed the number of subpasses in this
+ * render pass minus 1.
+ */
+ assert(subpass_id < cmd_state->pass->subpass_count);
+ return subpass_id;
+}
+
#define ANV_DEFINE_HANDLE_CASTS(__anv_type, __VkType) \
\
static inline struct __anv_type * \
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_query_pool, VkQueryPool)
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_render_pass, VkRenderPass)
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_sampler, VkSampler)
+ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_semaphore, VkSemaphore)
ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_shader_module, VkShaderModule)
+ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_debug_report_callback, VkDebugReportCallbackEXT)
/* Gen-specific function declarations */
#ifdef genX
# define genX(x) gen9_##x
# include "anv_genX.h"
# undef genX
+# define genX(x) gen10_##x
+# include "anv_genX.h"
+# undef genX
#endif
#endif /* ANV_PRIVATE_H */