2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #define __gen_validate_value(x) VALGRIND_CHECK_MEM_IS_DEFINED(&(x), sizeof(x))
42 #include "brw_device_info.h"
43 #include "util/macros.h"
44 #include "util/list.h"
47 #include <vulkan/vulkan.h>
48 #include <vulkan/vulkan_intel.h>
49 #include <vulkan/vk_wsi_swapchain.h>
50 #include <vulkan/vk_wsi_device_swapchain.h>
52 #include "anv_entrypoints.h"
54 #include "brw_context.h"
60 #define anv_noreturn __attribute__((__noreturn__))
61 #define anv_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
63 #define MIN(a, b) ((a) < (b) ? (a) : (b))
64 #define MAX(a, b) ((a) > (b) ? (a) : (b))
66 static inline uint32_t
67 align_u32(uint32_t v
, uint32_t a
)
69 return (v
+ a
- 1) & ~(a
- 1);
73 align_i32(int32_t v
, int32_t a
)
75 return (v
+ a
- 1) & ~(a
- 1);
78 /** Alignment must be a power of 2. */
80 anv_is_aligned(uintmax_t n
, uintmax_t a
)
82 assert(a
== (a
& -a
));
83 return (n
& (a
- 1)) == 0;
86 static inline uint32_t
87 anv_minify(uint32_t n
, uint32_t levels
)
92 return MAX(n
>> levels
, 1);
96 anv_clear_mask(uint32_t *inout_mask
, uint32_t clear_mask
)
98 if (*inout_mask
& clear_mask
) {
99 *inout_mask
&= ~clear_mask
;
106 #define for_each_bit(b, dword) \
107 for (uint32_t __dword = (dword); \
108 (b) = __builtin_ffs(__dword) - 1, __dword; \
109 __dword &= ~(1 << (b)))
111 /* Define no kernel as 1, since that's an illegal offset for a kernel */
115 VkStructureType sType
;
119 /* Whenever we generate an error, pass it through this function. Useful for
120 * debugging, where we can break on it. Only call at error site, not when
121 * propagating errors. Might be useful to plug in a stack trace here.
124 VkResult
__vk_errorf(VkResult error
, const char *file
, int line
, const char *format
, ...);
127 #define vk_error(error) __vk_errorf(error, __FILE__, __LINE__, NULL);
128 #define vk_errorf(error, format, ...) __vk_errorf(error, __FILE__, __LINE__, format, ## __VA_ARGS__);
130 #define vk_error(error) error
133 void __anv_finishme(const char *file
, int line
, const char *format
, ...)
134 anv_printflike(3, 4);
135 void anv_loge(const char *format
, ...) anv_printflike(1, 2);
136 void anv_loge_v(const char *format
, va_list va
);
139 * Print a FINISHME message, including its source location.
141 #define anv_finishme(format, ...) \
142 __anv_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__);
144 /* A non-fatal assert. Useful for debugging. */
146 #define anv_assert(x) ({ \
147 if (unlikely(!(x))) \
148 fprintf(stderr, "%s:%d ASSERT: %s\n", __FILE__, __LINE__, #x); \
151 #define anv_assert(x)
154 void anv_abortf(const char *format
, ...) anv_noreturn
anv_printflike(1, 2);
155 void anv_abortfv(const char *format
, va_list va
) anv_noreturn
;
157 #define stub_return(v) \
159 anv_finishme("stub %s", __func__); \
165 anv_finishme("stub %s", __func__); \
170 * A dynamically growable, circular buffer. Elements are added at head and
171 * removed from tail. head and tail are free-running uint32_t indices and we
172 * only compute the modulo with size when accessing the array. This way,
173 * number of bytes in the queue is always head - tail, even in case of
180 uint32_t element_size
;
185 int anv_vector_init(struct anv_vector
*queue
, uint32_t element_size
, uint32_t size
);
186 void *anv_vector_add(struct anv_vector
*queue
);
187 void *anv_vector_remove(struct anv_vector
*queue
);
190 anv_vector_length(struct anv_vector
*queue
)
192 return (queue
->head
- queue
->tail
) / queue
->element_size
;
196 anv_vector_finish(struct anv_vector
*queue
)
201 #define anv_vector_foreach(elem, queue) \
202 static_assert(__builtin_types_compatible_p(__typeof__(queue), struct anv_vector *), ""); \
203 for (uint32_t __anv_vector_offset = (queue)->tail; \
204 elem = (queue)->data + (__anv_vector_offset & ((queue)->size - 1)), __anv_vector_offset < (queue)->head; \
205 __anv_vector_offset += (queue)->element_size)
210 /* Index into the current validation list. This is used by the
211 * validation list building alrogithm to track which buffers are already
212 * in the validation list so that we can ensure uniqueness.
216 /* Last known offset. This value is provided by the kernel when we
217 * execbuf and is used as the presumed offset for the next bunch of
226 /* Represents a lock-free linked list of "free" things. This is used by
227 * both the block pool and the state pools. Unfortunately, in order to
228 * solve the ABA problem, we can't use a single uint32_t head.
230 union anv_free_list
{
234 /* A simple count that is incremented every time the head changes. */
240 #define ANV_FREE_LIST_EMPTY ((union anv_free_list) { { 1, 0 } })
242 struct anv_block_state
{
252 struct anv_block_pool
{
253 struct anv_device
*device
;
260 * Array of mmaps and gem handles owned by the block pool, reclaimed when
261 * the block pool is destroyed.
263 struct anv_vector mmap_cleanups
;
267 union anv_free_list free_list
;
268 struct anv_block_state state
;
271 /* Block pools are backed by a fixed-size 2GB memfd */
272 #define BLOCK_POOL_MEMFD_SIZE (1ull << 32)
274 static inline uint32_t
275 anv_block_pool_size(struct anv_block_pool
*pool
)
277 return pool
->state
.end
;
286 struct anv_fixed_size_state_pool
{
288 union anv_free_list free_list
;
289 struct anv_block_state block
;
292 #define ANV_MIN_STATE_SIZE_LOG2 6
293 #define ANV_MAX_STATE_SIZE_LOG2 10
295 #define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2)
297 struct anv_state_pool
{
298 struct anv_block_pool
*block_pool
;
299 struct anv_fixed_size_state_pool buckets
[ANV_STATE_BUCKETS
];
302 struct anv_state_stream
{
303 struct anv_block_pool
*block_pool
;
305 uint32_t current_block
;
309 void anv_block_pool_init(struct anv_block_pool
*pool
,
310 struct anv_device
*device
, uint32_t block_size
);
311 void anv_block_pool_finish(struct anv_block_pool
*pool
);
312 uint32_t anv_block_pool_alloc(struct anv_block_pool
*pool
);
313 void anv_block_pool_free(struct anv_block_pool
*pool
, uint32_t offset
);
314 void anv_state_pool_init(struct anv_state_pool
*pool
,
315 struct anv_block_pool
*block_pool
);
316 void anv_state_pool_finish(struct anv_state_pool
*pool
);
317 struct anv_state
anv_state_pool_alloc(struct anv_state_pool
*pool
,
318 size_t state_size
, size_t alignment
);
319 void anv_state_pool_free(struct anv_state_pool
*pool
, struct anv_state state
);
320 void anv_state_stream_init(struct anv_state_stream
*stream
,
321 struct anv_block_pool
*block_pool
);
322 void anv_state_stream_finish(struct anv_state_stream
*stream
);
323 struct anv_state
anv_state_stream_alloc(struct anv_state_stream
*stream
,
324 uint32_t size
, uint32_t alignment
);
327 * Implements a pool of re-usable BOs. The interface is identical to that
328 * of block_pool except that each block is its own BO.
331 struct anv_device
*device
;
338 void anv_bo_pool_init(struct anv_bo_pool
*pool
,
339 struct anv_device
*device
, uint32_t block_size
);
340 void anv_bo_pool_finish(struct anv_bo_pool
*pool
);
341 VkResult
anv_bo_pool_alloc(struct anv_bo_pool
*pool
, struct anv_bo
*bo
);
342 void anv_bo_pool_free(struct anv_bo_pool
*pool
, const struct anv_bo
*bo
);
344 struct anv_physical_device
{
345 struct anv_instance
* instance
;
349 const struct brw_device_info
* info
;
350 uint64_t aperture_size
;
353 struct anv_instance
{
354 void * pAllocUserData
;
355 PFN_vkAllocFunction pfnAlloc
;
356 PFN_vkFreeFunction pfnFree
;
358 uint32_t physicalDeviceCount
;
359 struct anv_physical_device physicalDevice
;
361 struct anv_wsi_implementation
* wsi_impl
[VK_PLATFORM_NUM_WSI
];
364 VkResult
anv_init_wsi(struct anv_instance
*instance
);
365 void anv_finish_wsi(struct anv_instance
*instance
);
367 struct anv_meta_state
{
373 /** Pipeline that blits from a 2D image. */
374 VkPipeline pipeline_2d_src
;
376 /** Pipeline that blits from a 3D image. */
377 VkPipeline pipeline_3d_src
;
379 VkPipelineLayout pipeline_layout
;
380 VkDescriptorSetLayout ds_layout
;
384 VkDynamicRasterState rs_state
;
385 VkDynamicColorBlendState cb_state
;
386 VkDynamicDepthStencilState ds_state
;
391 struct anv_device
* device
;
393 struct anv_state_pool
* pool
;
396 * Serial number of the most recently completed batch executed on the
399 struct anv_state completed_serial
;
402 * The next batch submitted to the engine will be assigned this serial
405 uint32_t next_serial
;
407 uint32_t last_collected_serial
;
411 struct anv_instance
* instance
;
413 struct brw_device_info info
;
417 struct anv_bo_pool batch_bo_pool
;
419 struct anv_block_pool dynamic_state_block_pool
;
420 struct anv_state_pool dynamic_state_pool
;
422 struct anv_block_pool instruction_block_pool
;
423 struct anv_block_pool surface_state_block_pool
;
424 struct anv_state_pool surface_state_pool
;
426 struct anv_meta_state meta_state
;
428 struct anv_state border_colors
;
430 struct anv_queue queue
;
432 struct anv_block_pool scratch_block_pool
;
434 struct anv_compiler
* compiler
;
435 pthread_mutex_t mutex
;
439 anv_instance_alloc(struct anv_instance
* instance
,
442 VkSystemAllocType allocType
);
445 anv_instance_free(struct anv_instance
* instance
,
449 anv_device_alloc(struct anv_device
* device
,
452 VkSystemAllocType allocType
);
455 anv_device_free(struct anv_device
* device
,
458 void* anv_gem_mmap(struct anv_device
*device
,
459 uint32_t gem_handle
, uint64_t offset
, uint64_t size
);
460 void anv_gem_munmap(void *p
, uint64_t size
);
461 uint32_t anv_gem_create(struct anv_device
*device
, size_t size
);
462 void anv_gem_close(struct anv_device
*device
, int gem_handle
);
463 int anv_gem_userptr(struct anv_device
*device
, void *mem
, size_t size
);
464 int anv_gem_wait(struct anv_device
*device
, int gem_handle
, int64_t *timeout_ns
);
465 int anv_gem_execbuffer(struct anv_device
*device
,
466 struct drm_i915_gem_execbuffer2
*execbuf
);
467 int anv_gem_set_tiling(struct anv_device
*device
, int gem_handle
,
468 uint32_t stride
, uint32_t tiling
);
469 int anv_gem_create_context(struct anv_device
*device
);
470 int anv_gem_destroy_context(struct anv_device
*device
, int context
);
471 int anv_gem_get_param(int fd
, uint32_t param
);
472 int anv_gem_get_aperture(int fd
, uint64_t *size
);
473 int anv_gem_handle_to_fd(struct anv_device
*device
, int gem_handle
);
474 int anv_gem_fd_to_handle(struct anv_device
*device
, int fd
);
475 int anv_gem_userptr(struct anv_device
*device
, void *mem
, size_t size
);
477 VkResult
anv_bo_init_new(struct anv_bo
*bo
, struct anv_device
*device
, uint64_t size
);
479 struct anv_reloc_list
{
482 struct drm_i915_gem_relocation_entry
* relocs
;
483 struct anv_bo
** reloc_bos
;
486 VkResult
anv_reloc_list_init(struct anv_reloc_list
*list
,
487 struct anv_device
*device
);
488 void anv_reloc_list_finish(struct anv_reloc_list
*list
,
489 struct anv_device
*device
);
491 uint64_t anv_reloc_list_add(struct anv_reloc_list
*list
,
492 struct anv_device
*device
,
493 uint32_t offset
, struct anv_bo
*target_bo
,
496 struct anv_batch_bo
{
497 /* Link in the anv_cmd_buffer.owned_batch_bos list */
498 struct list_head link
;
502 /* Bytes actually consumed in this batch BO */
505 struct anv_reloc_list relocs
;
509 struct anv_device
* device
;
515 struct anv_reloc_list
* relocs
;
517 /* This callback is called (with the associated user data) in the event
518 * that the batch runs out of space.
520 VkResult (*extend_cb
)(struct anv_batch
*, void *);
524 void *anv_batch_emit_dwords(struct anv_batch
*batch
, int num_dwords
);
525 void anv_batch_emit_batch(struct anv_batch
*batch
, struct anv_batch
*other
);
526 uint64_t anv_batch_emit_reloc(struct anv_batch
*batch
,
527 void *location
, struct anv_bo
*bo
, uint32_t offset
);
534 #define __gen_address_type struct anv_address
535 #define __gen_user_data struct anv_batch
537 static inline uint64_t
538 __gen_combine_address(struct anv_batch
*batch
, void *location
,
539 const struct anv_address address
, uint32_t delta
)
541 if (address
.bo
== NULL
) {
542 return address
.offset
+ delta
;
544 assert(batch
->start
<= location
&& location
< batch
->end
);
546 return anv_batch_emit_reloc(batch
, location
, address
.bo
, address
.offset
+ delta
);
550 #include "gen7_pack.h"
551 #include "gen75_pack.h"
552 #undef GEN8_3DSTATE_MULTISAMPLE
553 #include "gen8_pack.h"
555 #define anv_batch_emit(batch, cmd, ...) do { \
556 void *__dst = anv_batch_emit_dwords(batch, cmd ## _length); \
557 struct cmd __template = { \
561 cmd ## _pack(batch, __dst, &__template); \
562 VG(VALGRIND_CHECK_MEM_IS_DEFINED(__dst, cmd ## _length * 4)); \
565 #define anv_batch_emitn(batch, n, cmd, ...) ({ \
566 void *__dst = anv_batch_emit_dwords(batch, n); \
567 struct cmd __template = { \
569 .DwordLength = n - cmd ## _length_bias, \
572 cmd ## _pack(batch, __dst, &__template); \
576 #define anv_batch_emit_merge(batch, dwords0, dwords1) \
580 assert(ARRAY_SIZE(dwords0) == ARRAY_SIZE(dwords1)); \
581 dw = anv_batch_emit_dwords((batch), ARRAY_SIZE(dwords0)); \
582 for (uint32_t i = 0; i < ARRAY_SIZE(dwords0); i++) \
583 dw[i] = (dwords0)[i] | (dwords1)[i]; \
584 VG(VALGRIND_CHECK_MEM_IS_DEFINED(dw, ARRAY_SIZE(dwords0) * 4));\
587 static const struct GEN7_MEMORY_OBJECT_CONTROL_STATE GEN7_MOCS
= {
588 .GraphicsDataTypeGFDT
= 0,
589 .LLCCacheabilityControlLLCCC
= 0,
590 .L3CacheabilityControlL3CC
= 1
593 #define GEN8_MOCS { \
594 .MemoryTypeLLCeLLCCacheabilityControl = WB, \
595 .TargetCache = L3DefertoPATforLLCeLLCselection, \
599 struct anv_device_memory
{
601 VkDeviceSize map_size
;
605 struct anv_dynamic_vp_state
{
606 struct anv_state sf_clip_vp
;
607 struct anv_state cc_vp
;
608 struct anv_state scissor
;
611 struct anv_dynamic_rs_state
{
613 uint32_t sf
[GEN7_3DSTATE_SF_length
];
617 uint32_t sf
[GEN8_3DSTATE_SF_length
];
618 uint32_t raster
[GEN8_3DSTATE_RASTER_length
];
622 struct anv_dynamic_ds_state
{
624 uint32_t depth_stencil_state
[GEN7_DEPTH_STENCIL_STATE_length
];
625 uint32_t color_calc_state
[GEN8_COLOR_CALC_STATE_length
];
629 uint32_t wm_depth_stencil
[GEN8_3DSTATE_WM_DEPTH_STENCIL_length
];
630 uint32_t color_calc_state
[GEN8_COLOR_CALC_STATE_length
];
634 struct anv_dynamic_cb_state
{
635 uint32_t color_calc_state
[GEN8_COLOR_CALC_STATE_length
];
639 struct anv_descriptor_slot
{
644 struct anv_descriptor_set_layout
{
646 uint32_t surface_count
;
647 struct anv_descriptor_slot
*surface_start
;
648 uint32_t sampler_count
;
649 struct anv_descriptor_slot
*sampler_start
;
650 } stage
[VK_SHADER_STAGE_NUM
];
653 uint32_t num_dynamic_buffers
;
654 uint32_t shader_stages
;
655 struct anv_descriptor_slot entries
[0];
658 struct anv_descriptor
{
659 struct anv_sampler
*sampler
;
660 struct anv_surface_view
*view
;
663 struct anv_descriptor_set
{
664 struct anv_descriptor descriptors
[0];
668 anv_descriptor_set_create(struct anv_device
*device
,
669 const struct anv_descriptor_set_layout
*layout
,
670 struct anv_descriptor_set
**out_set
);
673 anv_descriptor_set_destroy(struct anv_device
*device
,
674 struct anv_descriptor_set
*set
);
679 #define MAX_PUSH_CONSTANTS_SIZE 128
680 #define MAX_DYNAMIC_BUFFERS 16
683 struct anv_pipeline_layout
{
685 struct anv_descriptor_set_layout
*layout
;
686 uint32_t dynamic_offset_start
;
688 uint32_t surface_start
;
689 uint32_t sampler_start
;
690 } stage
[VK_SHADER_STAGE_NUM
];
696 bool has_dynamic_offsets
;
697 uint32_t surface_count
;
698 uint32_t sampler_count
;
699 } stage
[VK_SHADER_STAGE_NUM
];
703 struct anv_device
* device
;
711 #define ANV_CMD_BUFFER_PIPELINE_DIRTY (1 << 0)
712 #define ANV_CMD_BUFFER_RS_DIRTY (1 << 2)
713 #define ANV_CMD_BUFFER_DS_DIRTY (1 << 3)
714 #define ANV_CMD_BUFFER_CB_DIRTY (1 << 4)
715 #define ANV_CMD_BUFFER_VP_DIRTY (1 << 5)
716 #define ANV_CMD_BUFFER_INDEX_BUFFER_DIRTY (1 << 6)
718 struct anv_vertex_binding
{
719 struct anv_buffer
* buffer
;
723 struct anv_descriptor_set_binding
{
724 struct anv_descriptor_set
* set
;
725 uint32_t dynamic_offsets
[128];
728 struct anv_push_constants
{
729 /* Current allocated size of this push constants data structure.
730 * Because a decent chunk of it may not be used (images on SKL, for
731 * instance), we won't actually allocate the entire structure up-front.
735 /* Push constant data provided by the client through vkPushConstants */
736 uint8_t client_data
[MAX_PUSH_CONSTANTS_SIZE
];
738 /* Our hardware only provides zero-based vertex and instance id so, in
739 * order to satisfy the vulkan requirements, we may have to push one or
740 * both of these into the shader.
742 uint32_t base_vertex
;
743 uint32_t base_instance
;
745 /* Offsets for dynamically bound buffers */
746 uint32_t dynamic_offsets
[MAX_DYNAMIC_BUFFERS
];
748 /* Image data for image_load_store on pre-SKL */
749 struct brw_image_param images
[MAX_IMAGES
];
752 /** State required while building cmd buffer */
753 struct anv_cmd_state
{
754 uint32_t current_pipeline
;
757 uint32_t compute_dirty
;
758 uint32_t descriptors_dirty
;
759 uint32_t push_constants_dirty
;
760 uint32_t scratch_size
;
761 struct anv_pipeline
* pipeline
;
762 struct anv_pipeline
* compute_pipeline
;
763 struct anv_framebuffer
* framebuffer
;
764 struct anv_render_pass
* pass
;
765 struct anv_subpass
* subpass
;
766 struct anv_dynamic_rs_state
* rs_state
;
767 struct anv_dynamic_ds_state
* ds_state
;
768 struct anv_dynamic_vp_state
* vp_state
;
769 struct anv_dynamic_cb_state
* cb_state
;
770 uint32_t state_vf
[GEN8_3DSTATE_VF_length
];
771 struct anv_vertex_binding vertex_bindings
[MAX_VBS
];
772 struct anv_descriptor_set_binding descriptors
[MAX_SETS
];
773 struct anv_push_constants
* push_constants
[VK_SHADER_STAGE_NUM
];
776 struct anv_buffer
* index_buffer
;
778 uint32_t index_offset
;
782 struct anv_cmd_pool
{
783 struct list_head cmd_buffers
;
786 #define ANV_CMD_BUFFER_BATCH_SIZE 8192
788 enum anv_cmd_buffer_exec_mode
{
789 ANV_CMD_BUFFER_EXEC_MODE_PRIMARY
,
790 ANV_CMD_BUFFER_EXEC_MODE_EMIT
,
791 ANV_CMD_BUFFER_EXEC_MODE_CHAIN
,
792 ANV_CMD_BUFFER_EXEC_MODE_COPY_AND_CHAIN
,
795 struct anv_cmd_buffer
{
796 struct anv_device
* device
;
798 struct list_head pool_link
;
800 struct anv_batch batch
;
802 /* Fields required for the actual chain of anv_batch_bo's.
804 * These fields are initialized by anv_cmd_buffer_init_batch_bo_chain().
806 struct list_head batch_bos
;
807 struct list_head surface_bos
;
808 uint32_t surface_next
;
809 enum anv_cmd_buffer_exec_mode exec_mode
;
811 /* A vector of anv_batch_bo pointers for every batch or surface buffer
812 * referenced by this command buffer
814 * initialized by anv_cmd_buffer_init_batch_bo_chain()
816 struct anv_vector seen_bbos
;
818 /* Information needed for execbuf
820 * These fields are generated by anv_cmd_buffer_prepare_execbuf().
823 struct drm_i915_gem_execbuffer2 execbuf
;
825 struct drm_i915_gem_exec_object2
* objects
;
827 struct anv_bo
** bos
;
829 /* Allocated length of the 'objects' and 'bos' arrays */
830 uint32_t array_length
;
835 /* Serial for tracking buffer completion */
838 /* Stream objects for storing temporary data */
839 struct anv_state_stream surface_state_stream
;
840 struct anv_state_stream dynamic_state_stream
;
842 VkCmdBufferOptimizeFlags opt_flags
;
843 VkCmdBufferLevel level
;
845 struct anv_cmd_state state
;
848 VkResult
anv_cmd_buffer_init_batch_bo_chain(struct anv_cmd_buffer
*cmd_buffer
);
849 void anv_cmd_buffer_fini_batch_bo_chain(struct anv_cmd_buffer
*cmd_buffer
);
850 void anv_cmd_buffer_reset_batch_bo_chain(struct anv_cmd_buffer
*cmd_buffer
);
851 void anv_cmd_buffer_end_batch_buffer(struct anv_cmd_buffer
*cmd_buffer
);
852 void anv_cmd_buffer_add_secondary(struct anv_cmd_buffer
*primary
,
853 struct anv_cmd_buffer
*secondary
);
854 void anv_cmd_buffer_prepare_execbuf(struct anv_cmd_buffer
*cmd_buffer
);
856 VkResult
anv_cmd_buffer_emit_binding_table(struct anv_cmd_buffer
*cmd_buffer
,
857 unsigned stage
, struct anv_state
*bt_state
);
858 VkResult
anv_cmd_buffer_emit_samplers(struct anv_cmd_buffer
*cmd_buffer
,
859 unsigned stage
, struct anv_state
*state
);
860 void anv_flush_descriptor_sets(struct anv_cmd_buffer
*cmd_buffer
);
862 struct anv_state
anv_cmd_buffer_emit_dynamic(struct anv_cmd_buffer
*cmd_buffer
,
863 uint32_t *a
, uint32_t dwords
,
865 struct anv_state
anv_cmd_buffer_merge_dynamic(struct anv_cmd_buffer
*cmd_buffer
,
866 uint32_t *a
, uint32_t *b
,
867 uint32_t dwords
, uint32_t alignment
);
868 void anv_cmd_buffer_begin_subpass(struct anv_cmd_buffer
*cmd_buffer
,
869 struct anv_subpass
*subpass
);
872 anv_cmd_buffer_current_surface_bo(struct anv_cmd_buffer
*cmd_buffer
);
873 struct anv_reloc_list
*
874 anv_cmd_buffer_current_surface_relocs(struct anv_cmd_buffer
*cmd_buffer
);
876 anv_cmd_buffer_alloc_surface_state(struct anv_cmd_buffer
*cmd_buffer
,
877 uint32_t size
, uint32_t alignment
);
879 anv_cmd_buffer_alloc_dynamic_state(struct anv_cmd_buffer
*cmd_buffer
,
880 uint32_t size
, uint32_t alignment
);
882 VkResult
anv_cmd_buffer_new_surface_state_bo(struct anv_cmd_buffer
*cmd_buffer
);
884 void gen7_cmd_buffer_emit_state_base_address(struct anv_cmd_buffer
*cmd_buffer
);
885 void gen8_cmd_buffer_emit_state_base_address(struct anv_cmd_buffer
*cmd_buffer
);
887 void anv_cmd_buffer_emit_state_base_address(struct anv_cmd_buffer
*cmd_buffer
);
889 void gen7_cmd_buffer_begin_subpass(struct anv_cmd_buffer
*cmd_buffer
,
890 struct anv_subpass
*subpass
);
892 void gen8_cmd_buffer_begin_subpass(struct anv_cmd_buffer
*cmd_buffer
,
893 struct anv_subpass
*subpass
);
895 void anv_cmd_buffer_begin_subpass(struct anv_cmd_buffer
*cmd_buffer
,
896 struct anv_subpass
*subpass
);
899 anv_cmd_buffer_push_constants(struct anv_cmd_buffer
*cmd_buffer
,
900 VkShaderStage stage
);
902 void anv_cmd_buffer_clear_attachments(struct anv_cmd_buffer
*cmd_buffer
,
903 struct anv_render_pass
*pass
,
904 const VkClearValue
*clear_values
);
905 const struct anv_depth_stencil_view
*
906 anv_cmd_buffer_get_depth_stencil_view(const struct anv_cmd_buffer
*cmd_buffer
);
908 void anv_cmd_buffer_dump(struct anv_cmd_buffer
*cmd_buffer
);
912 struct drm_i915_gem_execbuffer2 execbuf
;
913 struct drm_i915_gem_exec_object2 exec2_objects
[1];
917 struct anv_shader_module
{
923 struct anv_shader_module
* module
;
927 struct anv_pipeline
{
928 struct anv_device
* device
;
929 struct anv_batch batch
;
930 uint32_t batch_data
[256];
931 struct anv_reloc_list batch_relocs
;
932 struct anv_shader
* shaders
[VK_SHADER_STAGE_NUM
];
933 struct anv_pipeline_layout
* layout
;
936 struct brw_vs_prog_data vs_prog_data
;
937 struct brw_wm_prog_data wm_prog_data
;
938 struct brw_gs_prog_data gs_prog_data
;
939 struct brw_cs_prog_data cs_prog_data
;
940 bool writes_point_size
;
941 struct brw_stage_prog_data
* prog_data
[VK_SHADER_STAGE_NUM
];
942 uint32_t scratch_start
[VK_SHADER_STAGE_NUM
];
943 uint32_t total_scratch
;
947 uint32_t nr_vs_entries
;
950 uint32_t nr_gs_entries
;
953 uint32_t active_stages
;
954 struct anv_state_stream program_stream
;
955 struct anv_state blend_state
;
962 uint32_t ps_grf_start0
;
963 uint32_t ps_grf_start2
;
965 uint32_t gs_vertex_count
;
969 uint32_t binding_stride
[MAX_VBS
];
970 bool instancing_enable
[MAX_VBS
];
971 bool primitive_restart
;
974 uint32_t cs_thread_width_max
;
975 uint32_t cs_right_mask
;
978 uint32_t sf
[GEN7_3DSTATE_SF_length
];
979 uint32_t depth_stencil_state
[GEN7_DEPTH_STENCIL_STATE_length
];
983 uint32_t sf
[GEN8_3DSTATE_SF_length
];
984 uint32_t vf
[GEN8_3DSTATE_VF_length
];
985 uint32_t raster
[GEN8_3DSTATE_RASTER_length
];
986 uint32_t wm_depth_stencil
[GEN8_3DSTATE_WM_DEPTH_STENCIL_length
];
990 struct anv_graphics_pipeline_create_info
{
992 bool disable_viewport
;
993 bool disable_scissor
;
999 anv_pipeline_init(struct anv_pipeline
*pipeline
, struct anv_device
*device
,
1000 const VkGraphicsPipelineCreateInfo
*pCreateInfo
,
1001 const struct anv_graphics_pipeline_create_info
*extra
);
1004 anv_graphics_pipeline_create(VkDevice device
,
1005 const VkGraphicsPipelineCreateInfo
*pCreateInfo
,
1006 const struct anv_graphics_pipeline_create_info
*extra
,
1007 VkPipeline
*pPipeline
);
1010 gen7_graphics_pipeline_create(VkDevice _device
,
1011 const VkGraphicsPipelineCreateInfo
*pCreateInfo
,
1012 const struct anv_graphics_pipeline_create_info
*extra
,
1013 VkPipeline
*pPipeline
);
1016 gen8_graphics_pipeline_create(VkDevice _device
,
1017 const VkGraphicsPipelineCreateInfo
*pCreateInfo
,
1018 const struct anv_graphics_pipeline_create_info
*extra
,
1019 VkPipeline
*pPipeline
);
1021 gen7_compute_pipeline_create(VkDevice _device
,
1022 const VkComputePipelineCreateInfo
*pCreateInfo
,
1023 VkPipeline
*pPipeline
);
1026 gen8_compute_pipeline_create(VkDevice _device
,
1027 const VkComputePipelineCreateInfo
*pCreateInfo
,
1028 VkPipeline
*pPipeline
);
1030 struct anv_compiler
*anv_compiler_create(struct anv_device
*device
);
1031 void anv_compiler_destroy(struct anv_compiler
*compiler
);
1032 int anv_compiler_run(struct anv_compiler
*compiler
, struct anv_pipeline
*pipeline
);
1033 void anv_compiler_free(struct anv_pipeline
*pipeline
);
1036 const VkFormat vk_format
;
1038 uint16_t surface_format
; /**< RENDER_SURFACE_STATE.SurfaceFormat */
1039 uint8_t cpp
; /**< Bytes-per-pixel of anv_format::surface_format. */
1040 uint8_t num_channels
;
1041 uint16_t depth_format
; /**< 3DSTATE_DEPTH_BUFFER.SurfaceFormat */
1046 * Stencil formats are often a special case. To reduce the number of lookups
1047 * into the VkFormat-to-anv_format translation table when working with
1048 * stencil, here is the handle to the table's entry for VK_FORMAT_S8_UINT.
1050 extern const struct anv_format
*const anv_format_s8_uint
;
1052 const struct anv_format
*
1053 anv_format_for_vk_format(VkFormat format
);
1056 anv_format_is_color(const struct anv_format
*format
)
1058 return !format
->depth_format
&& !format
->has_stencil
;
1062 anv_format_is_depth_or_stencil(const struct anv_format
*format
)
1064 return format
->depth_format
|| format
->has_stencil
;
1067 struct anv_image_view_info
{
1068 uint8_t surface_type
; /**< RENDER_SURFACE_STATE.SurfaceType */
1069 bool is_array
:1; /**< RENDER_SURFACE_STATE.SurfaceArray */
1070 bool is_cube
:1; /**< RENDER_SURFACE_STATE.CubeFaceEnable* */
1073 const struct anv_image_view_info
*
1074 anv_image_view_info_for_vk_image_view_type(VkImageViewType type
);
1077 * A proxy for the color surfaces, depth surfaces, and stencil surfaces.
1079 struct anv_surface
{
1081 * Offset from VkImage's base address, as bound by vkBindImageMemory().
1085 uint32_t stride
; /**< RENDER_SURFACE_STATE.SurfacePitch */
1086 uint16_t qpitch
; /**< RENDER_SURFACE_STATE.QPitch */
1089 * \name Alignment of miptree images, in units of pixels.
1091 * These fields contain the real alignment values, not the values to be
1092 * given to the GPU. For example, if h_align is 4, then program the GPU
1096 uint8_t h_align
; /**< RENDER_SURFACE_STATE.SurfaceHorizontalAlignment */
1097 uint8_t v_align
; /**< RENDER_SURFACE_STATE.SurfaceVerticalAlignment */
1100 uint8_t tile_mode
; /**< RENDER_SURFACE_STATE.TileMode */
1105 const struct anv_format
*format
;
1108 uint32_t array_size
;
1113 /* Set when bound */
1115 VkDeviceSize offset
;
1117 /** RENDER_SURFACE_STATE.SurfaceType */
1123 * For each foo, anv_image::foo_surface is valid if and only if
1124 * anv_image::format has a foo aspect.
1126 * The hardware requires that the depth buffer and stencil buffer be
1127 * separate surfaces. From Vulkan's perspective, though, depth and stencil
1128 * reside in the same VkImage. To satisfy both the hardware and Vulkan, we
1129 * allocate the depth and stencil buffers as separate surfaces in the same
1133 struct anv_surface color_surface
;
1136 struct anv_surface depth_surface
;
1137 struct anv_surface stencil_surface
;
1142 struct anv_surface_view
{
1143 struct anv_state surface_state
; /**< RENDER_SURFACE_STATE */
1145 uint32_t offset
; /**< VkBufferCreateInfo::offset */
1146 uint32_t range
; /**< VkBufferCreateInfo::range */
1147 const struct anv_format
*format
; /**< VkBufferCreateInfo::format */
1150 struct anv_buffer_view
{
1151 struct anv_surface_view view
;
1154 struct anv_image_view
{
1155 struct anv_surface_view view
;
1159 enum anv_attachment_view_type
{
1160 ANV_ATTACHMENT_VIEW_TYPE_COLOR
,
1161 ANV_ATTACHMENT_VIEW_TYPE_DEPTH_STENCIL
,
1164 struct anv_attachment_view
{
1165 enum anv_attachment_view_type attachment_type
;
1169 struct anv_color_attachment_view
{
1170 struct anv_attachment_view base
;
1171 struct anv_surface_view view
;
1174 struct anv_depth_stencil_view
{
1175 struct anv_attachment_view base
;
1176 const struct anv_image
*image
; /**< VkAttachmentViewCreateInfo::image */
1177 const struct anv_format
*format
; /**< VkAttachmentViewCreateInfo::format */
1180 struct anv_image_create_info
{
1181 const VkImageCreateInfo
*vk_info
;
1182 bool force_tile_mode
;
1187 VkResult
anv_image_create(VkDevice _device
,
1188 const struct anv_image_create_info
*info
,
1191 struct anv_surface
*
1192 anv_image_get_surface_for_aspect(struct anv_image
*image
, VkImageAspect aspect
);
1194 struct anv_surface
*
1195 anv_image_get_surface_for_color_attachment(struct anv_image
*image
);
1197 void anv_image_view_init(struct anv_image_view
*view
,
1198 struct anv_device
*device
,
1199 const VkImageViewCreateInfo
* pCreateInfo
,
1200 struct anv_cmd_buffer
*cmd_buffer
);
1203 gen7_image_view_init(struct anv_image_view
*iview
,
1204 struct anv_device
*device
,
1205 const VkImageViewCreateInfo
* pCreateInfo
,
1206 struct anv_cmd_buffer
*cmd_buffer
);
1209 gen8_image_view_init(struct anv_image_view
*iview
,
1210 struct anv_device
*device
,
1211 const VkImageViewCreateInfo
* pCreateInfo
,
1212 struct anv_cmd_buffer
*cmd_buffer
);
1214 void anv_color_attachment_view_init(struct anv_color_attachment_view
*view
,
1215 struct anv_device
*device
,
1216 const VkAttachmentViewCreateInfo
* pCreateInfo
,
1217 struct anv_cmd_buffer
*cmd_buffer
);
1219 void gen7_color_attachment_view_init(struct anv_color_attachment_view
*aview
,
1220 struct anv_device
*device
,
1221 const VkAttachmentViewCreateInfo
* pCreateInfo
,
1222 struct anv_cmd_buffer
*cmd_buffer
);
1224 void gen8_color_attachment_view_init(struct anv_color_attachment_view
*aview
,
1225 struct anv_device
*device
,
1226 const VkAttachmentViewCreateInfo
* pCreateInfo
,
1227 struct anv_cmd_buffer
*cmd_buffer
);
1229 VkResult
anv_buffer_view_create(struct anv_device
*device
,
1230 const VkBufferViewCreateInfo
*pCreateInfo
,
1231 struct anv_buffer_view
**view_out
);
1233 void anv_fill_buffer_surface_state(struct anv_device
*device
, void *state
,
1234 const struct anv_format
*format
,
1235 uint32_t offset
, uint32_t range
);
1237 void gen7_fill_buffer_surface_state(void *state
, const struct anv_format
*format
,
1238 uint32_t offset
, uint32_t range
);
1239 void gen8_fill_buffer_surface_state(void *state
, const struct anv_format
*format
,
1240 uint32_t offset
, uint32_t range
);
1242 void anv_surface_view_fini(struct anv_device
*device
,
1243 struct anv_surface_view
*view
);
1245 struct anv_sampler
{
1249 struct anv_framebuffer
{
1254 /* Viewport for clears */
1255 VkDynamicViewportState vp_state
;
1257 uint32_t attachment_count
;
1258 const struct anv_attachment_view
* attachments
[0];
1261 struct anv_subpass
{
1262 uint32_t input_count
;
1263 uint32_t * input_attachments
;
1264 uint32_t color_count
;
1265 uint32_t * color_attachments
;
1266 uint32_t * resolve_attachments
;
1267 uint32_t depth_stencil_attachment
;
1270 struct anv_render_pass_attachment
{
1271 const struct anv_format
*format
;
1273 VkAttachmentLoadOp load_op
;
1274 VkAttachmentLoadOp stencil_load_op
;
1277 struct anv_render_pass
{
1278 uint32_t attachment_count
;
1279 uint32_t subpass_count
;
1281 uint32_t num_color_clear_attachments
;
1282 bool has_depth_clear_attachment
;
1283 bool has_stencil_clear_attachment
;
1285 struct anv_render_pass_attachment
* attachments
;
1286 struct anv_subpass subpasses
[0];
1289 struct anv_query_pool_slot
{
1295 struct anv_query_pool
{
1301 void anv_device_init_meta(struct anv_device
*device
);
1302 void anv_device_finish_meta(struct anv_device
*device
);
1304 void *anv_lookup_entrypoint(const char *name
);
1306 #define ANV_DEFINE_HANDLE_CASTS(__anv_type, __VkType) \
1308 static inline struct __anv_type * \
1309 __anv_type ## _from_handle(__VkType _handle) \
1311 return (struct __anv_type *) _handle; \
1314 static inline __VkType \
1315 __anv_type ## _to_handle(struct __anv_type *_obj) \
1317 return (__VkType) _obj; \
1320 #define ANV_DEFINE_NONDISP_HANDLE_CASTS(__anv_type, __VkType) \
1322 static inline struct __anv_type * \
1323 __anv_type ## _from_handle(__VkType _handle) \
1325 return (struct __anv_type *) _handle.handle; \
1328 static inline __VkType \
1329 __anv_type ## _to_handle(struct __anv_type *_obj) \
1331 return (__VkType) { .handle = (uint64_t) _obj }; \
1334 #define ANV_FROM_HANDLE(__anv_type, __name, __handle) \
1335 struct __anv_type *__name = __anv_type ## _from_handle(__handle)
1337 ANV_DEFINE_HANDLE_CASTS(anv_cmd_buffer
, VkCmdBuffer
)
1338 ANV_DEFINE_HANDLE_CASTS(anv_device
, VkDevice
)
1339 ANV_DEFINE_HANDLE_CASTS(anv_instance
, VkInstance
)
1340 ANV_DEFINE_HANDLE_CASTS(anv_physical_device
, VkPhysicalDevice
)
1341 ANV_DEFINE_HANDLE_CASTS(anv_queue
, VkQueue
)
1343 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_cmd_pool
, VkCmdPool
)
1344 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_attachment_view
, VkAttachmentView
)
1345 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_buffer
, VkBuffer
)
1346 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_buffer_view
, VkBufferView
);
1347 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_descriptor_set
, VkDescriptorSet
)
1348 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_descriptor_set_layout
, VkDescriptorSetLayout
)
1349 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_device_memory
, VkDeviceMemory
)
1350 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_dynamic_cb_state
, VkDynamicColorBlendState
)
1351 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_dynamic_ds_state
, VkDynamicDepthStencilState
)
1352 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_dynamic_rs_state
, VkDynamicRasterState
)
1353 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_dynamic_vp_state
, VkDynamicViewportState
)
1354 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_fence
, VkFence
)
1355 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_framebuffer
, VkFramebuffer
)
1356 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_image
, VkImage
)
1357 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_image_view
, VkImageView
);
1358 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_pipeline
, VkPipeline
)
1359 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_pipeline_layout
, VkPipelineLayout
)
1360 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_query_pool
, VkQueryPool
)
1361 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_render_pass
, VkRenderPass
)
1362 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_sampler
, VkSampler
)
1363 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_shader
, VkShader
)
1364 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_shader_module
, VkShaderModule
)
1366 #define ANV_DEFINE_STRUCT_CASTS(__anv_type, __VkType) \
1368 static inline const __VkType * \
1369 __anv_type ## _to_ ## __VkType(const struct __anv_type *__anv_obj) \
1371 return (const __VkType *) __anv_obj; \
1374 #define ANV_COMMON_TO_STRUCT(__VkType, __vk_name, __common_name) \
1375 const __VkType *__vk_name = anv_common_to_ ## __VkType(__common_name)
1377 ANV_DEFINE_STRUCT_CASTS(anv_common
, VkMemoryBarrier
)
1378 ANV_DEFINE_STRUCT_CASTS(anv_common
, VkBufferMemoryBarrier
)
1379 ANV_DEFINE_STRUCT_CASTS(anv_common
, VkImageMemoryBarrier
)