2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
33 #include "brw_device_info.h"
34 #include "util/macros.h"
37 #include <vulkan/vulkan.h>
38 #include <vulkan/vulkan_intel.h>
39 #include <vulkan/vk_wsi_lunarg.h>
41 #include "entrypoints.h"
43 #include "brw_context.h"
49 static inline uint32_t
50 ALIGN_U32(uint32_t v
, uint32_t a
)
52 return (v
+ a
- 1) & ~(a
- 1);
56 ALIGN_I32(int32_t v
, int32_t a
)
58 return (v
+ a
- 1) & ~(a
- 1);
61 #define for_each_bit(b, dword) \
62 for (uint32_t __dword = (dword); \
63 (b) = __builtin_ffs(__dword) - 1, __dword; \
64 __dword &= ~(1 << (b)))
66 /* Define no kernel as 1, since that's an illegal offset for a kernel */
70 VkStructureType sType
;
74 /* Whenever we generate an error, pass it through this function. Useful for
75 * debugging, where we can break on it. Only call at error site, not when
76 * propagating errors. Might be useful to plug in a stack trace here.
79 static inline VkResult
80 vk_error(VkResult error
)
83 fprintf(stderr
, "vk_error: %x\n", error
);
89 void __anv_finishme(const char *file
, int line
, const char *format
, ...);
92 * Print a FINISHME message, including its source location.
94 #define anv_finishme(format, ...) \
95 __anv_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__);
97 #define stub_return(v) \
99 anv_finishme("stub %s", __func__); \
105 anv_finishme("stub %s", __func__); \
110 * A dynamically growable, circular buffer. Elements are added at head and
111 * removed from tail. head and tail are free-running uint32_t indices and we
112 * only compute the modulo with size when accessing the array. This way,
113 * number of bytes in the queue is always head - tail, even in case of
120 uint32_t element_size
;
125 int anv_vector_init(struct anv_vector
*queue
, uint32_t element_size
, uint32_t size
);
126 void *anv_vector_add(struct anv_vector
*queue
);
127 void *anv_vector_remove(struct anv_vector
*queue
);
130 anv_vector_length(struct anv_vector
*queue
)
132 return (queue
->head
- queue
->tail
) / queue
->element_size
;
136 anv_vector_finish(struct anv_vector
*queue
)
141 #define anv_vector_foreach(elem, queue) \
142 static_assert(__builtin_types_compatible_p(__typeof__(queue), struct anv_vector *), ""); \
143 for (uint32_t __anv_vector_offset = (queue)->tail; \
144 elem = (queue)->data + (__anv_vector_offset & ((queue)->size - 1)), __anv_vector_offset < (queue)->head; \
145 __anv_vector_offset += (queue)->element_size)
153 /* This field is here for the benefit of the aub dumper. It can (and for
154 * userptr bos it must) be set to the cpu map of the buffer. Destroying
155 * the bo won't clean up the mmap, it's still the responsibility of the bo
156 * user to do that. */
160 /* Represents a lock-free linked list of "free" things. This is used by
161 * both the block pool and the state pools. Unfortunately, in order to
162 * solve the ABA problem, we can't use a single uint32_t head.
164 union anv_free_list
{
168 /* A simple count that is incremented every time the head changes. */
174 #define ANV_FREE_LIST_EMPTY ((union anv_free_list) { { 1, 0 } })
176 struct anv_block_pool
{
177 struct anv_device
*device
;
185 * Array of mmaps and gem handles owned by the block pool, reclaimed when
186 * the block pool is destroyed.
188 struct anv_vector mmap_cleanups
;
193 union anv_free_list free_list
;
196 struct anv_block_state
{
212 struct anv_fixed_size_state_pool
{
214 union anv_free_list free_list
;
215 struct anv_block_state block
;
218 #define ANV_MIN_STATE_SIZE_LOG2 6
219 #define ANV_MAX_STATE_SIZE_LOG2 10
221 #define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2)
223 struct anv_state_pool
{
224 struct anv_block_pool
*block_pool
;
225 struct anv_fixed_size_state_pool buckets
[ANV_STATE_BUCKETS
];
228 struct anv_state_stream
{
229 struct anv_block_pool
*block_pool
;
231 uint32_t current_block
;
235 void anv_block_pool_init(struct anv_block_pool
*pool
,
236 struct anv_device
*device
, uint32_t block_size
);
237 void anv_block_pool_init_slave(struct anv_block_pool
*pool
,
238 struct anv_block_pool
*master_pool
,
239 uint32_t num_blocks
);
240 void anv_block_pool_finish(struct anv_block_pool
*pool
);
241 uint32_t anv_block_pool_alloc(struct anv_block_pool
*pool
);
242 void anv_block_pool_free(struct anv_block_pool
*pool
, uint32_t offset
);
243 void anv_state_pool_init(struct anv_state_pool
*pool
,
244 struct anv_block_pool
*block_pool
);
245 struct anv_state
anv_state_pool_alloc(struct anv_state_pool
*pool
,
246 size_t state_size
, size_t alignment
);
247 void anv_state_pool_free(struct anv_state_pool
*pool
, struct anv_state state
);
248 void anv_state_stream_init(struct anv_state_stream
*stream
,
249 struct anv_block_pool
*block_pool
);
250 void anv_state_stream_finish(struct anv_state_stream
*stream
);
251 struct anv_state
anv_state_stream_alloc(struct anv_state_stream
*stream
,
252 uint32_t size
, uint32_t alignment
);
257 typedef void (*anv_object_destructor_cb
)(struct anv_device
*,
262 anv_object_destructor_cb destructor
;
265 struct anv_physical_device
{
266 struct anv_instance
* instance
;
271 const struct brw_device_info
* info
;
274 struct anv_instance
{
275 void * pAllocUserData
;
276 PFN_vkAllocFunction pfnAlloc
;
277 PFN_vkFreeFunction pfnFree
;
279 uint32_t physicalDeviceCount
;
280 struct anv_physical_device physicalDevice
;
283 struct anv_clear_state
{
285 VkDynamicRsState rs_state
;
286 VkDynamicCbState cb_state
;
289 struct anv_blit_state
{
291 VkDynamicRsState rs_state
;
292 VkDescriptorSetLayout ds_layout
;
293 VkDynamicCbState cb_state
;
297 struct anv_instance
* instance
;
299 struct brw_device_info info
;
305 struct anv_block_pool dynamic_state_block_pool
;
306 struct anv_state_pool dynamic_state_pool
;
308 struct anv_block_pool instruction_block_pool
;
309 struct anv_block_pool surface_state_block_pool
;
310 struct anv_block_pool binding_table_block_pool
;
311 struct anv_state_pool surface_state_pool
;
313 struct anv_clear_state clear_state
;
314 struct anv_blit_state blit_state
;
316 struct anv_compiler
* compiler
;
317 struct anv_aub_writer
* aub_writer
;
318 pthread_mutex_t mutex
;
322 struct anv_device
* device
;
324 struct anv_state_pool
* pool
;
327 * Serial number of the most recently completed batch executed on the
330 struct anv_state completed_serial
;
333 * The next batch submitted to the engine will be assigned this serial
336 uint32_t next_serial
;
338 uint32_t last_collected_serial
;
342 anv_device_alloc(struct anv_device
* device
,
345 VkSystemAllocType allocType
);
348 anv_device_free(struct anv_device
* device
,
351 void* anv_gem_mmap(struct anv_device
*device
,
352 uint32_t gem_handle
, uint64_t offset
, uint64_t size
);
353 void anv_gem_munmap(void *p
, uint64_t size
);
354 uint32_t anv_gem_create(struct anv_device
*device
, size_t size
);
355 void anv_gem_close(struct anv_device
*device
, int gem_handle
);
356 int anv_gem_userptr(struct anv_device
*device
, void *mem
, size_t size
);
357 int anv_gem_wait(struct anv_device
*device
, int gem_handle
, int64_t *timeout_ns
);
358 int anv_gem_execbuffer(struct anv_device
*device
,
359 struct drm_i915_gem_execbuffer2
*execbuf
);
360 int anv_gem_set_tiling(struct anv_device
*device
, int gem_handle
,
361 uint32_t stride
, uint32_t tiling
);
362 int anv_gem_create_context(struct anv_device
*device
);
363 int anv_gem_destroy_context(struct anv_device
*device
, int context
);
364 int anv_gem_get_param(int fd
, uint32_t param
);
365 int anv_gem_get_aperture(struct anv_device
*device
, uint64_t *size
);
366 int anv_gem_handle_to_fd(struct anv_device
*device
, int gem_handle
);
367 int anv_gem_fd_to_handle(struct anv_device
*device
, int fd
);
368 int anv_gem_userptr(struct anv_device
*device
, void *mem
, size_t size
);
370 VkResult
anv_bo_init_new(struct anv_bo
*bo
, struct anv_device
*device
, uint64_t size
);
372 /* TODO: Remove hardcoded reloc limit. */
373 #define ANV_BATCH_MAX_RELOCS 256
375 struct anv_reloc_list
{
377 struct drm_i915_gem_relocation_entry relocs
[ANV_BATCH_MAX_RELOCS
];
378 struct anv_bo
* reloc_bos
[ANV_BATCH_MAX_RELOCS
];
384 struct anv_reloc_list cmd_relocs
;
387 VkResult
anv_batch_init(struct anv_batch
*batch
, struct anv_device
*device
);
388 void anv_batch_finish(struct anv_batch
*batch
, struct anv_device
*device
);
389 void anv_batch_reset(struct anv_batch
*batch
);
390 void *anv_batch_emit_dwords(struct anv_batch
*batch
, int num_dwords
);
391 void anv_batch_emit_batch(struct anv_batch
*batch
, struct anv_batch
*other
);
392 uint64_t anv_batch_emit_reloc(struct anv_batch
*batch
,
393 void *location
, struct anv_bo
*bo
, uint32_t offset
);
400 #define __gen_address_type struct anv_address
401 #define __gen_user_data struct anv_batch
403 static inline uint64_t
404 __gen_combine_address(struct anv_batch
*batch
, void *location
,
405 const struct anv_address address
, uint32_t delta
)
407 if (address
.bo
== NULL
) {
410 assert(batch
->bo
.map
<= location
&&
411 (char *) location
< (char *) batch
->bo
.map
+ batch
->bo
.size
);
413 return anv_batch_emit_reloc(batch
, location
, address
.bo
, address
.offset
+ delta
);
417 #include "gen7_pack.h"
418 #include "gen75_pack.h"
419 #undef GEN8_3DSTATE_MULTISAMPLE
420 #include "gen8_pack.h"
422 #define anv_batch_emit(batch, cmd, ...) do { \
423 struct cmd __template = { \
427 void *__dst = anv_batch_emit_dwords(batch, cmd ## _length); \
428 cmd ## _pack(batch, __dst, &__template); \
431 #define anv_batch_emitn(batch, n, cmd, ...) ({ \
432 struct cmd __template = { \
434 .DwordLength = n - cmd ## _length_bias, \
437 void *__dst = anv_batch_emit_dwords(batch, n); \
438 cmd ## _pack(batch, __dst, &__template); \
442 #define anv_batch_emit_merge(batch, dwords0, dwords1) \
446 assert(ARRAY_SIZE(dwords0) == ARRAY_SIZE(dwords1)); \
447 dw = anv_batch_emit_dwords((batch), ARRAY_SIZE(dwords0)); \
448 for (uint32_t i = 0; i < ARRAY_SIZE(dwords0); i++) \
449 dw[i] = (dwords0)[i] | (dwords1)[i]; \
452 #define GEN8_MOCS { \
453 .MemoryTypeLLCeLLCCacheabilityControl = WB, \
454 .TargetCache = L3DefertoPATforLLCeLLCselection, \
458 struct anv_device_memory
{
460 VkDeviceSize map_size
;
464 struct anv_dynamic_vp_state
{
465 struct anv_object base
;
466 struct anv_state sf_clip_vp
;
467 struct anv_state cc_vp
;
468 struct anv_state scissor
;
471 struct anv_dynamic_rs_state
{
472 uint32_t state_sf
[GEN8_3DSTATE_SF_length
];
473 uint32_t state_raster
[GEN8_3DSTATE_RASTER_length
];
476 struct anv_dynamic_ds_state
{
477 uint32_t state_wm_depth_stencil
[GEN8_3DSTATE_WM_DEPTH_STENCIL_length
];
478 uint32_t state_color_calc
[GEN8_COLOR_CALC_STATE_length
];
481 struct anv_dynamic_cb_state
{
482 uint32_t state_color_calc
[GEN8_COLOR_CALC_STATE_length
];
486 struct anv_query_pool_slot
{
492 struct anv_query_pool
{
493 struct anv_object base
;
499 struct anv_descriptor_set_layout
{
501 uint32_t surface_count
;
502 uint32_t *surface_start
;
503 uint32_t sampler_count
;
504 uint32_t *sampler_start
;
505 } stage
[VK_NUM_SHADER_STAGE
];
508 uint32_t num_dynamic_buffers
;
512 struct anv_descriptor
{
513 struct anv_sampler
*sampler
;
514 struct anv_surface_view
*view
;
517 struct anv_descriptor_set
{
518 struct anv_descriptor descriptors
[0];
525 struct anv_pipeline_layout
{
527 struct anv_descriptor_set_layout
*layout
;
528 uint32_t surface_start
[VK_NUM_SHADER_STAGE
];
529 uint32_t sampler_start
[VK_NUM_SHADER_STAGE
];
535 uint32_t surface_count
;
536 uint32_t sampler_count
;
537 } stage
[VK_NUM_SHADER_STAGE
];
541 struct anv_device
* device
;
549 #define ANV_CMD_BUFFER_PIPELINE_DIRTY (1 << 0)
550 #define ANV_CMD_BUFFER_DESCRIPTOR_SET_DIRTY (1 << 1)
551 #define ANV_CMD_BUFFER_RS_DIRTY (1 << 2)
552 #define ANV_CMD_BUFFER_DS_DIRTY (1 << 3)
553 #define ANV_CMD_BUFFER_CB_DIRTY (1 << 4)
555 struct anv_bindings
{
557 struct anv_buffer
*buffer
;
562 uint32_t surfaces
[256];
563 struct { uint32_t dwords
[4]; } samplers
[16];
564 } descriptors
[VK_NUM_SHADER_STAGE
];
567 struct anv_cmd_buffer
{
568 struct anv_object base
;
569 struct anv_device
* device
;
571 struct drm_i915_gem_execbuffer2 execbuf
;
572 struct drm_i915_gem_exec_object2
* exec2_objects
;
573 struct anv_bo
** exec2_bos
;
578 struct anv_batch batch
;
579 struct anv_bo surface_bo
;
580 uint32_t surface_next
;
581 struct anv_reloc_list surface_relocs
;
582 struct anv_state_stream binding_table_state_stream
;
583 struct anv_state_stream surface_state_stream
;
584 struct anv_state_stream dynamic_state_stream
;
586 /* State required while building cmd buffer */
589 struct anv_pipeline
* pipeline
;
590 struct anv_framebuffer
* framebuffer
;
591 struct anv_dynamic_rs_state
* rs_state
;
592 struct anv_dynamic_ds_state
* ds_state
;
593 struct anv_dynamic_vp_state
* vp_state
;
594 struct anv_dynamic_cb_state
* cb_state
;
595 struct anv_bindings
* bindings
;
596 struct anv_bindings default_bindings
;
599 void anv_cmd_buffer_dump(struct anv_cmd_buffer
*cmd_buffer
);
600 void anv_aub_writer_destroy(struct anv_aub_writer
*writer
);
603 struct anv_object base
;
605 struct drm_i915_gem_execbuffer2 execbuf
;
606 struct drm_i915_gem_exec_object2 exec2_objects
[1];
615 struct anv_pipeline
{
616 struct anv_object base
;
617 struct anv_device
* device
;
618 struct anv_batch batch
;
619 struct anv_shader
* shaders
[VK_NUM_SHADER_STAGE
];
620 struct anv_pipeline_layout
* layout
;
623 struct brw_vs_prog_data vs_prog_data
;
624 struct brw_wm_prog_data wm_prog_data
;
625 struct brw_gs_prog_data gs_prog_data
;
626 struct brw_stage_prog_data
* prog_data
[VK_NUM_SHADER_STAGE
];
630 uint32_t nr_vs_entries
;
633 uint32_t nr_gs_entries
;
636 struct anv_bo vs_scratch_bo
;
637 struct anv_bo ps_scratch_bo
;
638 struct anv_bo gs_scratch_bo
;
640 uint32_t active_stages
;
641 struct anv_state_stream program_stream
;
642 struct anv_state blend_state
;
647 uint32_t gs_vertex_count
;
650 uint32_t binding_stride
[MAX_VBS
];
652 uint32_t state_sf
[GEN8_3DSTATE_SF_length
];
653 uint32_t state_raster
[GEN8_3DSTATE_RASTER_length
];
654 uint32_t state_wm_depth_stencil
[GEN8_3DSTATE_WM_DEPTH_STENCIL_length
];
657 struct anv_pipeline_create_info
{
659 bool disable_viewport
;
660 bool disable_scissor
;
666 anv_pipeline_create(VkDevice device
,
667 const VkGraphicsPipelineCreateInfo
*pCreateInfo
,
668 const struct anv_pipeline_create_info
*extra
,
669 VkPipeline
*pPipeline
);
671 struct anv_compiler
*anv_compiler_create(int fd
);
672 void anv_compiler_destroy(struct anv_compiler
*compiler
);
673 int anv_compiler_run(struct anv_compiler
*compiler
, struct anv_pipeline
*pipeline
);
674 void anv_compiler_free(struct anv_pipeline
*pipeline
);
683 const struct anv_format
*
684 anv_format_for_vk_format(VkFormat format
);
695 uint32_t stencil_offset
;
696 uint32_t stencil_stride
;
702 struct anv_swap_chain
* swap_chain
;
705 struct anv_surface_view
{
706 struct anv_state surface_state
;
713 struct anv_image_create_info
{
717 VkResult
anv_image_create(VkDevice _device
,
718 const VkImageCreateInfo
*pCreateInfo
,
719 const struct anv_image_create_info
*extra
,
722 void anv_image_view_init(struct anv_surface_view
*view
,
723 struct anv_device
*device
,
724 const VkImageViewCreateInfo
* pCreateInfo
,
725 struct anv_cmd_buffer
*cmd_buffer
);
727 void anv_color_attachment_view_init(struct anv_surface_view
*view
,
728 struct anv_device
*device
,
729 const VkColorAttachmentViewCreateInfo
* pCreateInfo
,
730 struct anv_cmd_buffer
*cmd_buffer
);
736 struct anv_depth_stencil_view
{
739 uint32_t depth_offset
;
740 uint32_t depth_stride
;
741 uint32_t depth_format
;
743 uint32_t stencil_offset
;
744 uint32_t stencil_stride
;
747 struct anv_framebuffer
{
748 struct anv_object base
;
749 uint32_t color_attachment_count
;
750 const struct anv_surface_view
* color_attachments
[MAX_RTS
];
751 const struct anv_depth_stencil_view
* depth_stencil
;
753 uint32_t sample_count
;
758 /* Viewport for clears */
759 VkDynamicVpState vp_state
;
762 struct anv_render_pass_layer
{
763 VkAttachmentLoadOp color_load_op
;
764 VkClearColor clear_color
;
767 struct anv_render_pass
{
770 uint32_t num_clear_layers
;
772 struct anv_render_pass_layer layers
[0];
775 void anv_device_init_meta(struct anv_device
*device
);
778 anv_cmd_buffer_clear(struct anv_cmd_buffer
*cmd_buffer
,
779 struct anv_render_pass
*pass
);
782 anv_cmd_buffer_fill_render_targets(struct anv_cmd_buffer
*cmd_buffer
);
785 anv_lookup_entrypoint(const char *name
);