#include <valgrind.h>
#include <memcheck.h>
#define VG(x) x
+#ifndef NDEBUG
#define __gen_validate_value(x) VALGRIND_CHECK_MEM_IS_DEFINED(&(x), sizeof(x))
+#endif
#else
#define VG(x)
#endif
#include "common/gen_clflush.h"
+#include "common/gen_gem.h"
#include "dev/gen_device_info.h"
#include "blorp/blorp.h"
#include "compiler/brw_compiler.h"
#include "util/macros.h"
+#include "util/hash_table.h"
#include "util/list.h"
+#include "util/set.h"
#include "util/u_atomic.h"
#include "util/u_vector.h"
+#include "util/vma.h"
#include "vk_alloc.h"
#include "vk_debug_report.h"
#include "common/intel_log.h"
#include "wsi_common.h"
+/* anv Virtual Memory Layout
+ * =========================
+ *
+ * When the anv driver is determining the virtual graphics addresses of memory
+ * objects itself using the softpin mechanism, the following memory ranges
+ * will be used.
+ *
+ * Three special considerations to notice:
+ *
+ * (1) the dynamic state pool is located within the same 4 GiB as the low
+ * heap. This is to work around a VF cache issue described in a comment in
+ * anv_physical_device_init_heaps.
+ *
+ * (2) the binding table pool is located at lower addresses than the surface
+ * state pool, within a 4 GiB range. This allows surface state base addresses
+ * to cover both binding tables (16 bit offsets) and surface states (32 bit
+ * offsets).
+ *
+ * (3) the last 4 GiB of the address space is withheld from the high
+ * heap. Various hardware units will read past the end of an object for
+ * various reasons. This healthy margin prevents reads from wrapping around
+ * 48-bit addresses.
+ */
+#define LOW_HEAP_MIN_ADDRESS 0x000000001000ULL /* 4 KiB */
+#define LOW_HEAP_MAX_ADDRESS 0x0000bfffffffULL
+#define DYNAMIC_STATE_POOL_MIN_ADDRESS 0x0000c0000000ULL /* 3 GiB */
+#define DYNAMIC_STATE_POOL_MAX_ADDRESS 0x0000ffffffffULL
+#define BINDING_TABLE_POOL_MIN_ADDRESS 0x000100000000ULL /* 4 GiB */
+#define BINDING_TABLE_POOL_MAX_ADDRESS 0x00013fffffffULL
+#define SURFACE_STATE_POOL_MIN_ADDRESS 0x000140000000ULL /* 5 GiB */
+#define SURFACE_STATE_POOL_MAX_ADDRESS 0x00017fffffffULL
+#define INSTRUCTION_STATE_POOL_MIN_ADDRESS 0x000180000000ULL /* 6 GiB */
+#define INSTRUCTION_STATE_POOL_MAX_ADDRESS 0x0001bfffffffULL
+#define HIGH_HEAP_MIN_ADDRESS 0x0001c0000000ULL /* 7 GiB */
+#define HIGH_HEAP_MAX_ADDRESS 0xfffeffffffffULL
+
+#define LOW_HEAP_SIZE \
+ (LOW_HEAP_MAX_ADDRESS - LOW_HEAP_MIN_ADDRESS + 1)
+#define HIGH_HEAP_SIZE \
+ (HIGH_HEAP_MAX_ADDRESS - HIGH_HEAP_MIN_ADDRESS + 1)
+#define DYNAMIC_STATE_POOL_SIZE \
+ (DYNAMIC_STATE_POOL_MAX_ADDRESS - DYNAMIC_STATE_POOL_MIN_ADDRESS + 1)
+#define BINDING_TABLE_POOL_SIZE \
+ (BINDING_TABLE_POOL_MAX_ADDRESS - BINDING_TABLE_POOL_MIN_ADDRESS + 1)
+#define SURFACE_STATE_POOL_SIZE \
+ (SURFACE_STATE_POOL_MAX_ADDRESS - SURFACE_STATE_POOL_MIN_ADDRESS + 1)
+#define INSTRUCTION_STATE_POOL_SIZE \
+ (INSTRUCTION_STATE_POOL_MAX_ADDRESS - INSTRUCTION_STATE_POOL_MIN_ADDRESS + 1)
+
/* Allowing different clear colors requires us to perform a depth resolve at
* the end of certain render passes. This is because while slow clears store
* the clear color in the HiZ buffer, fast clears (without a resolve) don't.
struct anv_bo bo;
+ /* The address where the start of the pool is pinned. The various bos that
+ * are created as the pool grows will have addresses in the range
+ * [start_address, start_address + BLOCK_POOL_MEMFD_SIZE).
+ */
+ uint64_t start_address;
+
/* The offset from the start of the bo to the "center" of the block
* pool. Pointers to allocated blocks are given by
* bo.map + center_bo_offset + offsets.
*/
VkResult anv_block_pool_init(struct anv_block_pool *pool,
struct anv_device *device,
+ uint64_t start_address,
uint32_t initial_size,
uint64_t bo_flags);
void anv_block_pool_finish(struct anv_block_pool *pool);
VkResult anv_state_pool_init(struct anv_state_pool *pool,
struct anv_device *device,
+ uint64_t start_address,
uint32_t block_size,
uint64_t bo_flags);
void anv_state_pool_finish(struct anv_state_pool *pool);
void anv_bo_cache_finish(struct anv_bo_cache *cache);
VkResult anv_bo_cache_alloc(struct anv_device *device,
struct anv_bo_cache *cache,
- uint64_t size, struct anv_bo **bo);
+ uint64_t size, uint64_t bo_flags,
+ struct anv_bo **bo);
VkResult anv_bo_cache_import(struct anv_device *device,
struct anv_bo_cache *cache,
- int fd, struct anv_bo **bo);
+ int fd, uint64_t bo_flags,
+ struct anv_bo **bo);
VkResult anv_bo_cache_export(struct anv_device *device,
struct anv_bo_cache *cache,
struct anv_bo *bo_in, int *fd_out);
bool has_syncobj;
bool has_syncobj_wait;
bool has_context_priority;
+ bool use_softpin;
+ bool has_context_isolation;
struct anv_device_extension_table supported_extensions;
struct anv_memory_heap heaps[VK_MAX_MEMORY_HEAPS];
} memory;
+ uint8_t driver_build_sha1[20];
uint8_t pipeline_cache_uuid[VK_UUID_SIZE];
uint8_t driver_uuid[VK_UUID_SIZE];
uint8_t device_uuid[VK_UUID_SIZE];
+ struct disk_cache * disk_cache;
+
struct wsi_device wsi_device;
int local_fd;
+ int master_fd;
};
struct anv_instance {
int physicalDeviceCount;
struct anv_physical_device physicalDevice;
+ bool pipeline_cache_enabled;
+
struct vk_debug_report_instance debug_report_callbacks;
};
struct anv_device * device;
- struct anv_state_pool * pool;
-
VkDeviceQueueCreateFlags flags;
};
anv_pipeline_cache_upload_kernel(struct anv_pipeline_cache *cache,
const void *key_data, uint32_t key_size,
const void *kernel_data, uint32_t kernel_size,
+ const void *constant_data,
+ uint32_t constant_data_size,
const struct brw_stage_prog_data *prog_data,
uint32_t prog_data_size,
const struct anv_pipeline_bind_map *bind_map);
+struct anv_shader_bin *
+anv_device_search_for_kernel(struct anv_device *device,
+ struct anv_pipeline_cache *cache,
+ const void *key_data, uint32_t key_size);
+
+struct anv_shader_bin *
+anv_device_upload_kernel(struct anv_device *device,
+ struct anv_pipeline_cache *cache,
+ const void *key_data, uint32_t key_size,
+ const void *kernel_data, uint32_t kernel_size,
+ const void *constant_data,
+ uint32_t constant_data_size,
+ const struct brw_stage_prog_data *prog_data,
+ uint32_t prog_data_size,
+ const struct anv_pipeline_bind_map *bind_map);
+
struct anv_device {
VK_LOADER_DATA _loader_data;
struct anv_device_extension_table enabled_extensions;
struct anv_dispatch_table dispatch;
+ pthread_mutex_t vma_mutex;
+ struct util_vma_heap vma_lo;
+ struct util_vma_heap vma_hi;
+ uint64_t vma_lo_available;
+ uint64_t vma_hi_available;
+
struct anv_bo_pool batch_bo_pool;
struct anv_bo_cache bo_cache;
struct anv_state_pool dynamic_state_pool;
struct anv_state_pool instruction_state_pool;
+ struct anv_state_pool binding_table_pool;
struct anv_state_pool surface_state_pool;
struct anv_bo workaround_bo;
struct anv_bo trivial_batch_bo;
+ struct anv_bo hiz_clear_bo;
- struct anv_pipeline_cache blorp_shader_cache;
+ struct anv_pipeline_cache default_pipeline_cache;
struct blorp_context blorp;
struct anv_state border_colors;
bool lost;
};
+static inline struct anv_state_pool *
+anv_binding_table_pool(struct anv_device *device)
+{
+ if (device->instance->physicalDevice.use_softpin)
+ return &device->binding_table_pool;
+ else
+ return &device->surface_state_pool;
+}
+
+static inline struct anv_state
+anv_binding_table_pool_alloc(struct anv_device *device) {
+ if (device->instance->physicalDevice.use_softpin)
+ return anv_state_pool_alloc(&device->binding_table_pool,
+ device->binding_table_pool.block_size, 0);
+ else
+ return anv_state_pool_alloc_back(&device->surface_state_pool);
+}
+
+static inline void
+anv_binding_table_pool_free(struct anv_device *device, struct anv_state state) {
+ anv_state_pool_free(anv_binding_table_pool(device), state);
+}
+
static void inline
anv_state_flush(struct anv_device *device, struct anv_state state)
{
int anv_gem_get_tiling(struct anv_device *device, uint32_t gem_handle);
bool anv_gem_get_bit6_swizzle(int fd, uint32_t tiling);
int anv_gem_get_aperture(int fd, uint64_t *size);
-bool anv_gem_supports_48b_addresses(int fd);
int anv_gem_gpu_get_reset_stats(struct anv_device *device,
uint32_t *active, uint32_t *pending);
int anv_gem_handle_to_fd(struct anv_device *device, uint32_t gem_handle);
uint32_t *handles, uint32_t num_handles,
int64_t abs_timeout_ns, bool wait_all);
+bool anv_vma_alloc(struct anv_device *device, struct anv_bo *bo);
+void anv_vma_free(struct anv_device *device, struct anv_bo *bo);
+
VkResult anv_bo_init_new(struct anv_bo *bo, struct anv_device *device, uint64_t size);
struct anv_reloc_list {
uint32_t array_length;
struct drm_i915_gem_relocation_entry * relocs;
struct anv_bo ** reloc_bos;
+ struct set * deps;
};
VkResult anv_reloc_list_init(struct anv_reloc_list *list,
uint32_t offset;
};
+#define ANV_NULL_ADDRESS ((struct anv_address) { NULL, 0 })
+
+static inline bool
+anv_address_is_null(struct anv_address addr)
+{
+ return addr.bo == NULL && addr.offset == 0;
+}
+
+static inline uint64_t
+anv_address_physical(struct anv_address addr)
+{
+ if (addr.bo && (addr.bo->flags & EXEC_OBJECT_PINNED))
+ return gen_canonical_address(addr.bo->offset + addr.offset);
+ else
+ return gen_canonical_address(addr.offset);
+}
+
+static inline struct anv_address
+anv_address_add(struct anv_address addr, uint64_t offset)
+{
+ addr.offset += offset;
+ return addr;
+}
+
+static inline void
+write_reloc(const struct anv_device *device, void *p, uint64_t v, bool flush)
+{
+ unsigned reloc_size = 0;
+ if (device->info.gen >= 8) {
+ reloc_size = sizeof(uint64_t);
+ *(uint64_t *)p = gen_canonical_address(v);
+ } else {
+ reloc_size = sizeof(uint32_t);
+ *(uint32_t *)p = v;
+ }
+
+ if (flush && !device->info.has_llc)
+ gen_flush_range(p, reloc_size);
+}
+
static inline uint64_t
_anv_combine_address(struct anv_batch *batch, void *location,
const struct anv_address address, uint32_t delta)
struct anv_buffer_view {
enum isl_format format; /**< VkBufferViewCreateInfo::format */
- struct anv_bo *bo;
- uint32_t offset; /**< Offset into bo. */
uint64_t range; /**< VkBufferViewCreateInfo::range */
+ struct anv_address address;
+
struct anv_state surface_state;
struct anv_state storage_surface_state;
struct anv_state writeonly_storage_surface_state;
struct anv_descriptor_pool *pool,
struct anv_descriptor_set *set);
+#define ANV_DESCRIPTOR_SET_SHADER_CONSTANTS (UINT8_MAX - 1)
#define ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS UINT8_MAX
struct anv_pipeline_binding {
VkBufferUsageFlags usage;
/* Set when bound */
- struct anv_bo * bo;
- VkDeviceSize offset;
+ struct anv_address address;
};
static inline uint64_t
*
* This address is relative to the start of the BO.
*/
- uint64_t address;
+ struct anv_address address;
/* Address of the aux surface, if any
*
- * This field is 0 if and only if no aux surface exists.
+ * This field is ANV_NULL_ADDRESS if and only if no aux surface exists.
*
- * This address is relative to the start of the BO. With the exception of
- * gen8, the bottom 12 bits of this address include extra aux information.
+ * With the exception of gen8, the bottom 12 bits of this address' offset
+ * include extra aux information.
*/
- uint64_t aux_address;
+ struct anv_address aux_address;
+ /* Address of the clear color, if any
+ *
+ * This address is relative to the start of the BO.
+ */
+ struct anv_address clear_address;
};
/**
ANV_FENCE_TYPE_NONE = 0,
ANV_FENCE_TYPE_BO,
ANV_FENCE_TYPE_SYNCOBJ,
+ ANV_FENCE_TYPE_WSI,
};
enum anv_bo_fence_state {
/** DRM syncobj handle for syncobj-based fences */
uint32_t syncobj;
+
+ /** WSI fence */
+ struct wsi_fence *fence_wsi;
};
};
struct anv_state kernel;
uint32_t kernel_size;
+ struct anv_state constant_data;
+ uint32_t constant_data_size;
+
const struct brw_stage_prog_data *prog_data;
uint32_t prog_data_size;
anv_shader_bin_create(struct anv_device *device,
const void *key, uint32_t key_size,
const void *kernel, uint32_t kernel_size,
+ const void *constant_data, uint32_t constant_data_size,
const struct brw_stage_prog_data *prog_data,
uint32_t prog_data_size, const void *prog_data_param,
const struct anv_pipeline_bind_map *bind_map);
if (aspect_mask & VK_IMAGE_ASPECT_PLANE_2_BIT)
planes++;
+ if ((aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) != 0 &&
+ (aspect_mask & VK_IMAGE_ASPECT_STENCIL_BIT) != 0)
+ planes++;
+
return planes;
}
/**
* BO associated with this plane, set when bound.
*/
- struct anv_bo *bo;
- VkDeviceSize bo_offset;
+ struct anv_address address;
/**
* When destroying the image, also free the bo.
assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
- return (struct anv_address) {
- .bo = image->planes[plane].bo,
- .offset = image->planes[plane].bo_offset +
- image->planes[plane].fast_clear_state_offset,
- };
+ return anv_address_add(image->planes[plane].address,
+ image->planes[plane].fast_clear_state_offset);
}
static inline struct anv_address
{
struct anv_address addr =
anv_image_get_clear_color_addr(device, image, aspect);
- addr.offset += device->isl_dev.ss.clear_value_size;
+
+ const unsigned clear_color_state_size = device->info.gen >= 10 ?
+ device->isl_dev.ss.clear_color_state_size :
+ device->isl_dev.ss.clear_value_size;
+ addr.offset += clear_color_state_size;
return addr;
}
const struct anv_image *image,
VkImageAspectFlagBits aspect,
uint32_t base_layer, uint32_t layer_count,
- enum isl_aux_op mcs_op, bool predicate);
+ enum isl_aux_op mcs_op, union isl_color_value *clear_value,
+ bool predicate);
void
anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
const struct anv_image *image,
VkImageAspectFlagBits aspect, uint32_t level,
uint32_t base_layer, uint32_t layer_count,
- enum isl_aux_op ccs_op, bool predicate);
+ enum isl_aux_op ccs_op, union isl_color_value *clear_value,
+ bool predicate);
void
anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
void anv_fill_buffer_surface_state(struct anv_device *device,
struct anv_state state,
enum isl_format format,
- uint32_t offset, uint32_t range,
- uint32_t stride);
+ struct anv_address address,
+ uint32_t range, uint32_t stride);
+
+static inline void
+anv_clear_color_from_att_state(union isl_color_value *clear_color,
+ const struct anv_attachment_state *att_state,
+ const struct anv_image_view *iview)
+{
+ const struct isl_format_layout *view_fmtl =
+ isl_format_get_layout(iview->planes[0].isl.format);
+
+#define COPY_CLEAR_COLOR_CHANNEL(c, i) \
+ if (view_fmtl->channels.c.bits) \
+ clear_color->u32[i] = att_state->clear_value.color.uint32[i]
+
+ COPY_CLEAR_COLOR_CHANNEL(r, 0);
+ COPY_CLEAR_COLOR_CHANNEL(g, 1);
+ COPY_CLEAR_COLOR_CHANNEL(b, 2);
+ COPY_CLEAR_COLOR_CHANNEL(a, 3);
+
+#undef COPY_CLEAR_COLOR_CHANNEL
+}
struct anv_ycbcr_conversion {
struct anv_subpass_attachment * color_attachments;
struct anv_subpass_attachment * resolve_attachments;
- struct anv_subpass_attachment depth_stencil_attachment;
+ struct anv_subpass_attachment * depth_stencil_attachment;
uint32_t view_mask;
projects / mesa.git / blobdiff