0b1e97c64b6b8a3105eb3f8bf1986fc16b4e50d8
[mesa.git] / src / intel / vulkan / anv_private.h
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #ifndef ANV_PRIVATE_H
25 #define ANV_PRIVATE_H
26
27 #include <stdlib.h>
28 #include <stdio.h>
29 #include <stdbool.h>
30 #include <pthread.h>
31 #include <assert.h>
32 #include <stdint.h>
33 #include "drm-uapi/i915_drm.h"
34
35 #ifdef HAVE_VALGRIND
36 #include <valgrind.h>
37 #include <memcheck.h>
38 #define VG(x) x
39 #ifndef NDEBUG
40 #define __gen_validate_value(x) VALGRIND_CHECK_MEM_IS_DEFINED(&(x), sizeof(x))
41 #endif
42 #else
43 #define VG(x) ((void)0)
44 #endif
45
46 #include "common/gen_clflush.h"
47 #include "common/gen_decoder.h"
48 #include "common/gen_gem.h"
49 #include "dev/gen_device_info.h"
50 #include "blorp/blorp.h"
51 #include "compiler/brw_compiler.h"
52 #include "util/bitset.h"
53 #include "util/macros.h"
54 #include "util/hash_table.h"
55 #include "util/list.h"
56 #include "util/sparse_array.h"
57 #include "util/u_atomic.h"
58 #include "util/u_vector.h"
59 #include "util/u_math.h"
60 #include "util/vma.h"
61 #include "util/xmlconfig.h"
62 #include "vk_alloc.h"
63 #include "vk_debug_report.h"
64
65 /* Pre-declarations needed for WSI entrypoints */
66 struct wl_surface;
67 struct wl_display;
68 typedef struct xcb_connection_t xcb_connection_t;
69 typedef uint32_t xcb_visualid_t;
70 typedef uint32_t xcb_window_t;
71
72 struct anv_batch;
73 struct anv_buffer;
74 struct anv_buffer_view;
75 struct anv_image_view;
76 struct anv_instance;
77
78 struct gen_aux_map_context;
79 struct gen_l3_config;
80 struct gen_perf_config;
81
82 #include <vulkan/vulkan.h>
83 #include <vulkan/vulkan_intel.h>
84 #include <vulkan/vk_icd.h>
85
86 #include "anv_android.h"
87 #include "anv_entrypoints.h"
88 #include "anv_extensions.h"
89 #include "isl/isl.h"
90
91 #include "dev/gen_debug.h"
92 #include "common/intel_log.h"
93 #include "wsi_common.h"
94
95 #define NSEC_PER_SEC 1000000000ull
96
97 /* anv Virtual Memory Layout
98 * =========================
99 *
100 * When the anv driver is determining the virtual graphics addresses of memory
101 * objects itself using the softpin mechanism, the following memory ranges
102 * will be used.
103 *
104 * Three special considerations to notice:
105 *
106 * (1) the dynamic state pool is located within the same 4 GiB as the low
107 * heap. This is to work around a VF cache issue described in a comment in
108 * anv_physical_device_init_heaps.
109 *
110 * (2) the binding table pool is located at lower addresses than the surface
111 * state pool, within a 4 GiB range. This allows surface state base addresses
112 * to cover both binding tables (16 bit offsets) and surface states (32 bit
113 * offsets).
114 *
115 * (3) the last 4 GiB of the address space is withheld from the high
116 * heap. Various hardware units will read past the end of an object for
117 * various reasons. This healthy margin prevents reads from wrapping around
118 * 48-bit addresses.
119 */
120 #define LOW_HEAP_MIN_ADDRESS 0x000000001000ULL /* 4 KiB */
121 #define LOW_HEAP_MAX_ADDRESS 0x0000bfffffffULL
122 #define DYNAMIC_STATE_POOL_MIN_ADDRESS 0x0000c0000000ULL /* 3 GiB */
123 #define DYNAMIC_STATE_POOL_MAX_ADDRESS 0x0000ffffffffULL
124 #define BINDING_TABLE_POOL_MIN_ADDRESS 0x000100000000ULL /* 4 GiB */
125 #define BINDING_TABLE_POOL_MAX_ADDRESS 0x00013fffffffULL
126 #define SURFACE_STATE_POOL_MIN_ADDRESS 0x000140000000ULL /* 5 GiB */
127 #define SURFACE_STATE_POOL_MAX_ADDRESS 0x00017fffffffULL
128 #define INSTRUCTION_STATE_POOL_MIN_ADDRESS 0x000180000000ULL /* 6 GiB */
129 #define INSTRUCTION_STATE_POOL_MAX_ADDRESS 0x0001bfffffffULL
130 #define HIGH_HEAP_MIN_ADDRESS 0x0001c0000000ULL /* 7 GiB */
131
132 #define LOW_HEAP_SIZE \
133 (LOW_HEAP_MAX_ADDRESS - LOW_HEAP_MIN_ADDRESS + 1)
134 #define DYNAMIC_STATE_POOL_SIZE \
135 (DYNAMIC_STATE_POOL_MAX_ADDRESS - DYNAMIC_STATE_POOL_MIN_ADDRESS + 1)
136 #define BINDING_TABLE_POOL_SIZE \
137 (BINDING_TABLE_POOL_MAX_ADDRESS - BINDING_TABLE_POOL_MIN_ADDRESS + 1)
138 #define SURFACE_STATE_POOL_SIZE \
139 (SURFACE_STATE_POOL_MAX_ADDRESS - SURFACE_STATE_POOL_MIN_ADDRESS + 1)
140 #define INSTRUCTION_STATE_POOL_SIZE \
141 (INSTRUCTION_STATE_POOL_MAX_ADDRESS - INSTRUCTION_STATE_POOL_MIN_ADDRESS + 1)
142
143 /* Allowing different clear colors requires us to perform a depth resolve at
144 * the end of certain render passes. This is because while slow clears store
145 * the clear color in the HiZ buffer, fast clears (without a resolve) don't.
146 * See the PRMs for examples describing when additional resolves would be
147 * necessary. To enable fast clears without requiring extra resolves, we set
148 * the clear value to a globally-defined one. We could allow different values
149 * if the user doesn't expect coherent data during or after a render passes
150 * (VK_ATTACHMENT_STORE_OP_DONT_CARE), but such users (aside from the CTS)
151 * don't seem to exist yet. In almost all Vulkan applications tested thus far,
152 * 1.0f seems to be the only value used. The only application that doesn't set
153 * this value does so through the usage of an seemingly uninitialized clear
154 * value.
155 */
156 #define ANV_HZ_FC_VAL 1.0f
157
158 #define MAX_VBS 28
159 #define MAX_XFB_BUFFERS 4
160 #define MAX_XFB_STREAMS 4
161 #define MAX_SETS 8
162 #define MAX_RTS 8
163 #define MAX_VIEWPORTS 16
164 #define MAX_SCISSORS 16
165 #define MAX_PUSH_CONSTANTS_SIZE 128
166 #define MAX_DYNAMIC_BUFFERS 16
167 #define MAX_IMAGES 64
168 #define MAX_PUSH_DESCRIPTORS 32 /* Minimum requirement */
169 #define MAX_INLINE_UNIFORM_BLOCK_SIZE 4096
170 #define MAX_INLINE_UNIFORM_BLOCK_DESCRIPTORS 32
171
172 /* From the Skylake PRM Vol. 7 "Binding Table Surface State Model":
173 *
174 * "The surface state model is used when a Binding Table Index (specified
175 * in the message descriptor) of less than 240 is specified. In this model,
176 * the Binding Table Index is used to index into the binding table, and the
177 * binding table entry contains a pointer to the SURFACE_STATE."
178 *
179 * Binding table values above 240 are used for various things in the hardware
180 * such as stateless, stateless with incoherent cache, SLM, and bindless.
181 */
182 #define MAX_BINDING_TABLE_SIZE 240
183
184 /* The kernel relocation API has a limitation of a 32-bit delta value
185 * applied to the address before it is written which, in spite of it being
186 * unsigned, is treated as signed . Because of the way that this maps to
187 * the Vulkan API, we cannot handle an offset into a buffer that does not
188 * fit into a signed 32 bits. The only mechanism we have for dealing with
189 * this at the moment is to limit all VkDeviceMemory objects to a maximum
190 * of 2GB each. The Vulkan spec allows us to do this:
191 *
192 * "Some platforms may have a limit on the maximum size of a single
193 * allocation. For example, certain systems may fail to create
194 * allocations with a size greater than or equal to 4GB. Such a limit is
195 * implementation-dependent, and if such a failure occurs then the error
196 * VK_ERROR_OUT_OF_DEVICE_MEMORY should be returned."
197 *
198 * We don't use vk_error here because it's not an error so much as an
199 * indication to the application that the allocation is too large.
200 */
201 #define MAX_MEMORY_ALLOCATION_SIZE (1ull << 31)
202
203 #define ANV_SVGS_VB_INDEX MAX_VBS
204 #define ANV_DRAWID_VB_INDEX (MAX_VBS + 1)
205
206 /* We reserve this MI ALU register for the purpose of handling predication.
207 * Other code which uses the MI ALU should leave it alone.
208 */
209 #define ANV_PREDICATE_RESULT_REG 0x2678 /* MI_ALU_REG15 */
210
211 /* For gen12 we set the streamout buffers using 4 separate commands
212 * (3DSTATE_SO_BUFFER_INDEX_*) instead of 3DSTATE_SO_BUFFER. However the layout
213 * of the 3DSTATE_SO_BUFFER_INDEX_* commands is identical to that of
214 * 3DSTATE_SO_BUFFER apart from the SOBufferIndex field, so for now we use the
215 * 3DSTATE_SO_BUFFER command, but change the 3DCommandSubOpcode.
216 * SO_BUFFER_INDEX_0_CMD is actually the 3DCommandSubOpcode for
217 * 3DSTATE_SO_BUFFER_INDEX_0.
218 */
219 #define SO_BUFFER_INDEX_0_CMD 0x60
220 #define anv_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
221
222 static inline uint32_t
223 align_down_npot_u32(uint32_t v, uint32_t a)
224 {
225 return v - (v % a);
226 }
227
228 static inline uint32_t
229 align_u32(uint32_t v, uint32_t a)
230 {
231 assert(a != 0 && a == (a & -a));
232 return (v + a - 1) & ~(a - 1);
233 }
234
235 static inline uint64_t
236 align_u64(uint64_t v, uint64_t a)
237 {
238 assert(a != 0 && a == (a & -a));
239 return (v + a - 1) & ~(a - 1);
240 }
241
242 static inline int32_t
243 align_i32(int32_t v, int32_t a)
244 {
245 assert(a != 0 && a == (a & -a));
246 return (v + a - 1) & ~(a - 1);
247 }
248
249 /** Alignment must be a power of 2. */
250 static inline bool
251 anv_is_aligned(uintmax_t n, uintmax_t a)
252 {
253 assert(a == (a & -a));
254 return (n & (a - 1)) == 0;
255 }
256
257 static inline uint32_t
258 anv_minify(uint32_t n, uint32_t levels)
259 {
260 if (unlikely(n == 0))
261 return 0;
262 else
263 return MAX2(n >> levels, 1);
264 }
265
266 static inline float
267 anv_clamp_f(float f, float min, float max)
268 {
269 assert(min < max);
270
271 if (f > max)
272 return max;
273 else if (f < min)
274 return min;
275 else
276 return f;
277 }
278
279 static inline bool
280 anv_clear_mask(uint32_t *inout_mask, uint32_t clear_mask)
281 {
282 if (*inout_mask & clear_mask) {
283 *inout_mask &= ~clear_mask;
284 return true;
285 } else {
286 return false;
287 }
288 }
289
290 static inline union isl_color_value
291 vk_to_isl_color(VkClearColorValue color)
292 {
293 return (union isl_color_value) {
294 .u32 = {
295 color.uint32[0],
296 color.uint32[1],
297 color.uint32[2],
298 color.uint32[3],
299 },
300 };
301 }
302
303 static inline void *anv_unpack_ptr(uintptr_t ptr, int bits, int *flags)
304 {
305 uintptr_t mask = (1ull << bits) - 1;
306 *flags = ptr & mask;
307 return (void *) (ptr & ~mask);
308 }
309
310 static inline uintptr_t anv_pack_ptr(void *ptr, int bits, int flags)
311 {
312 uintptr_t value = (uintptr_t) ptr;
313 uintptr_t mask = (1ull << bits) - 1;
314 return value | (mask & flags);
315 }
316
317 #define for_each_bit(b, dword) \
318 for (uint32_t __dword = (dword); \
319 (b) = __builtin_ffs(__dword) - 1, __dword; \
320 __dword &= ~(1 << (b)))
321
322 #define typed_memcpy(dest, src, count) ({ \
323 STATIC_ASSERT(sizeof(*src) == sizeof(*dest)); \
324 memcpy((dest), (src), (count) * sizeof(*(src))); \
325 })
326
327 /* Mapping from anv object to VkDebugReportObjectTypeEXT. New types need
328 * to be added here in order to utilize mapping in debug/error/perf macros.
329 */
330 #define REPORT_OBJECT_TYPE(o) \
331 __builtin_choose_expr ( \
332 __builtin_types_compatible_p (__typeof (o), struct anv_instance*), \
333 VK_DEBUG_REPORT_OBJECT_TYPE_INSTANCE_EXT, \
334 __builtin_choose_expr ( \
335 __builtin_types_compatible_p (__typeof (o), struct anv_physical_device*), \
336 VK_DEBUG_REPORT_OBJECT_TYPE_PHYSICAL_DEVICE_EXT, \
337 __builtin_choose_expr ( \
338 __builtin_types_compatible_p (__typeof (o), struct anv_device*), \
339 VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT, \
340 __builtin_choose_expr ( \
341 __builtin_types_compatible_p (__typeof (o), const struct anv_device*), \
342 VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT, \
343 __builtin_choose_expr ( \
344 __builtin_types_compatible_p (__typeof (o), struct anv_queue*), \
345 VK_DEBUG_REPORT_OBJECT_TYPE_QUEUE_EXT, \
346 __builtin_choose_expr ( \
347 __builtin_types_compatible_p (__typeof (o), struct anv_semaphore*), \
348 VK_DEBUG_REPORT_OBJECT_TYPE_SEMAPHORE_EXT, \
349 __builtin_choose_expr ( \
350 __builtin_types_compatible_p (__typeof (o), struct anv_cmd_buffer*), \
351 VK_DEBUG_REPORT_OBJECT_TYPE_COMMAND_BUFFER_EXT, \
352 __builtin_choose_expr ( \
353 __builtin_types_compatible_p (__typeof (o), struct anv_fence*), \
354 VK_DEBUG_REPORT_OBJECT_TYPE_FENCE_EXT, \
355 __builtin_choose_expr ( \
356 __builtin_types_compatible_p (__typeof (o), struct anv_device_memory*), \
357 VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_MEMORY_EXT, \
358 __builtin_choose_expr ( \
359 __builtin_types_compatible_p (__typeof (o), struct anv_buffer*), \
360 VK_DEBUG_REPORT_OBJECT_TYPE_BUFFER_EXT, \
361 __builtin_choose_expr ( \
362 __builtin_types_compatible_p (__typeof (o), struct anv_image*), \
363 VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_EXT, \
364 __builtin_choose_expr ( \
365 __builtin_types_compatible_p (__typeof (o), const struct anv_image*), \
366 VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_EXT, \
367 __builtin_choose_expr ( \
368 __builtin_types_compatible_p (__typeof (o), struct anv_event*), \
369 VK_DEBUG_REPORT_OBJECT_TYPE_EVENT_EXT, \
370 __builtin_choose_expr ( \
371 __builtin_types_compatible_p (__typeof (o), struct anv_query_pool*), \
372 VK_DEBUG_REPORT_OBJECT_TYPE_QUERY_POOL_EXT, \
373 __builtin_choose_expr ( \
374 __builtin_types_compatible_p (__typeof (o), struct anv_buffer_view*), \
375 VK_DEBUG_REPORT_OBJECT_TYPE_BUFFER_VIEW_EXT, \
376 __builtin_choose_expr ( \
377 __builtin_types_compatible_p (__typeof (o), struct anv_image_view*), \
378 VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_VIEW_EXT, \
379 __builtin_choose_expr ( \
380 __builtin_types_compatible_p (__typeof (o), struct anv_shader_module*), \
381 VK_DEBUG_REPORT_OBJECT_TYPE_SHADER_MODULE_EXT, \
382 __builtin_choose_expr ( \
383 __builtin_types_compatible_p (__typeof (o), struct anv_pipeline_cache*), \
384 VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_CACHE_EXT, \
385 __builtin_choose_expr ( \
386 __builtin_types_compatible_p (__typeof (o), struct anv_pipeline_layout*), \
387 VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_LAYOUT_EXT, \
388 __builtin_choose_expr ( \
389 __builtin_types_compatible_p (__typeof (o), struct anv_render_pass*), \
390 VK_DEBUG_REPORT_OBJECT_TYPE_RENDER_PASS_EXT, \
391 __builtin_choose_expr ( \
392 __builtin_types_compatible_p (__typeof (o), struct anv_pipeline*), \
393 VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_EXT, \
394 __builtin_choose_expr ( \
395 __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_set_layout*), \
396 VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT_EXT, \
397 __builtin_choose_expr ( \
398 __builtin_types_compatible_p (__typeof (o), struct anv_sampler*), \
399 VK_DEBUG_REPORT_OBJECT_TYPE_SAMPLER_EXT, \
400 __builtin_choose_expr ( \
401 __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_pool*), \
402 VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_POOL_EXT, \
403 __builtin_choose_expr ( \
404 __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_set*), \
405 VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_SET_EXT, \
406 __builtin_choose_expr ( \
407 __builtin_types_compatible_p (__typeof (o), struct anv_framebuffer*), \
408 VK_DEBUG_REPORT_OBJECT_TYPE_FRAMEBUFFER_EXT, \
409 __builtin_choose_expr ( \
410 __builtin_types_compatible_p (__typeof (o), struct anv_cmd_pool*), \
411 VK_DEBUG_REPORT_OBJECT_TYPE_COMMAND_POOL_EXT, \
412 __builtin_choose_expr ( \
413 __builtin_types_compatible_p (__typeof (o), struct anv_surface*), \
414 VK_DEBUG_REPORT_OBJECT_TYPE_SURFACE_KHR_EXT, \
415 __builtin_choose_expr ( \
416 __builtin_types_compatible_p (__typeof (o), struct wsi_swapchain*), \
417 VK_DEBUG_REPORT_OBJECT_TYPE_SWAPCHAIN_KHR_EXT, \
418 __builtin_choose_expr ( \
419 __builtin_types_compatible_p (__typeof (o), struct vk_debug_callback*), \
420 VK_DEBUG_REPORT_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT_EXT, \
421 __builtin_choose_expr ( \
422 __builtin_types_compatible_p (__typeof (o), void*), \
423 VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT, \
424 /* The void expression results in a compile-time error \
425 when assigning the result to something. */ \
426 (void)0)))))))))))))))))))))))))))))))
427
428 /* Whenever we generate an error, pass it through this function. Useful for
429 * debugging, where we can break on it. Only call at error site, not when
430 * propagating errors. Might be useful to plug in a stack trace here.
431 */
432
433 VkResult __vk_errorv(struct anv_instance *instance, const void *object,
434 VkDebugReportObjectTypeEXT type, VkResult error,
435 const char *file, int line, const char *format,
436 va_list args);
437
438 VkResult __vk_errorf(struct anv_instance *instance, const void *object,
439 VkDebugReportObjectTypeEXT type, VkResult error,
440 const char *file, int line, const char *format, ...)
441 anv_printflike(7, 8);
442
443 #ifdef DEBUG
444 #define vk_error(error) __vk_errorf(NULL, NULL,\
445 VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT,\
446 error, __FILE__, __LINE__, NULL)
447 #define vk_errorv(instance, obj, error, format, args)\
448 __vk_errorv(instance, obj, REPORT_OBJECT_TYPE(obj), error,\
449 __FILE__, __LINE__, format, args)
450 #define vk_errorf(instance, obj, error, format, ...)\
451 __vk_errorf(instance, obj, REPORT_OBJECT_TYPE(obj), error,\
452 __FILE__, __LINE__, format, ## __VA_ARGS__)
453 #else
454 #define vk_error(error) error
455 #define vk_errorf(instance, obj, error, format, ...) error
456 #endif
457
458 /**
459 * Warn on ignored extension structs.
460 *
461 * The Vulkan spec requires us to ignore unsupported or unknown structs in
462 * a pNext chain. In debug mode, emitting warnings for ignored structs may
463 * help us discover structs that we should not have ignored.
464 *
465 *
466 * From the Vulkan 1.0.38 spec:
467 *
468 * Any component of the implementation (the loader, any enabled layers,
469 * and drivers) must skip over, without processing (other than reading the
470 * sType and pNext members) any chained structures with sType values not
471 * defined by extensions supported by that component.
472 */
473 #define anv_debug_ignored_stype(sType) \
474 intel_logd("%s: ignored VkStructureType %u\n", __func__, (sType))
475
476 void __anv_perf_warn(struct anv_instance *instance, const void *object,
477 VkDebugReportObjectTypeEXT type, const char *file,
478 int line, const char *format, ...)
479 anv_printflike(6, 7);
480 void anv_loge(const char *format, ...) anv_printflike(1, 2);
481 void anv_loge_v(const char *format, va_list va);
482
483 /**
484 * Print a FINISHME message, including its source location.
485 */
486 #define anv_finishme(format, ...) \
487 do { \
488 static bool reported = false; \
489 if (!reported) { \
490 intel_logw("%s:%d: FINISHME: " format, __FILE__, __LINE__, \
491 ##__VA_ARGS__); \
492 reported = true; \
493 } \
494 } while (0)
495
496 /**
497 * Print a perf warning message. Set INTEL_DEBUG=perf to see these.
498 */
499 #define anv_perf_warn(instance, obj, format, ...) \
500 do { \
501 static bool reported = false; \
502 if (!reported && unlikely(INTEL_DEBUG & DEBUG_PERF)) { \
503 __anv_perf_warn(instance, obj, REPORT_OBJECT_TYPE(obj), __FILE__, __LINE__,\
504 format, ##__VA_ARGS__); \
505 reported = true; \
506 } \
507 } while (0)
508
509 /* A non-fatal assert. Useful for debugging. */
510 #ifdef DEBUG
511 #define anv_assert(x) ({ \
512 if (unlikely(!(x))) \
513 intel_loge("%s:%d ASSERT: %s", __FILE__, __LINE__, #x); \
514 })
515 #else
516 #define anv_assert(x)
517 #endif
518
519 /* A multi-pointer allocator
520 *
521 * When copying data structures from the user (such as a render pass), it's
522 * common to need to allocate data for a bunch of different things. Instead
523 * of doing several allocations and having to handle all of the error checking
524 * that entails, it can be easier to do a single allocation. This struct
525 * helps facilitate that. The intended usage looks like this:
526 *
527 * ANV_MULTIALLOC(ma)
528 * anv_multialloc_add(&ma, &main_ptr, 1);
529 * anv_multialloc_add(&ma, &substruct1, substruct1Count);
530 * anv_multialloc_add(&ma, &substruct2, substruct2Count);
531 *
532 * if (!anv_multialloc_alloc(&ma, pAllocator, VK_ALLOCATION_SCOPE_FOO))
533 * return vk_error(VK_ERROR_OUT_OF_HOST_MEORY);
534 */
535 struct anv_multialloc {
536 size_t size;
537 size_t align;
538
539 uint32_t ptr_count;
540 void **ptrs[8];
541 };
542
543 #define ANV_MULTIALLOC_INIT \
544 ((struct anv_multialloc) { 0, })
545
546 #define ANV_MULTIALLOC(_name) \
547 struct anv_multialloc _name = ANV_MULTIALLOC_INIT
548
549 __attribute__((always_inline))
550 static inline void
551 _anv_multialloc_add(struct anv_multialloc *ma,
552 void **ptr, size_t size, size_t align)
553 {
554 size_t offset = align_u64(ma->size, align);
555 ma->size = offset + size;
556 ma->align = MAX2(ma->align, align);
557
558 /* Store the offset in the pointer. */
559 *ptr = (void *)(uintptr_t)offset;
560
561 assert(ma->ptr_count < ARRAY_SIZE(ma->ptrs));
562 ma->ptrs[ma->ptr_count++] = ptr;
563 }
564
565 #define anv_multialloc_add_size(_ma, _ptr, _size) \
566 _anv_multialloc_add((_ma), (void **)(_ptr), (_size), __alignof__(**(_ptr)))
567
568 #define anv_multialloc_add(_ma, _ptr, _count) \
569 anv_multialloc_add_size(_ma, _ptr, (_count) * sizeof(**(_ptr)));
570
571 __attribute__((always_inline))
572 static inline void *
573 anv_multialloc_alloc(struct anv_multialloc *ma,
574 const VkAllocationCallbacks *alloc,
575 VkSystemAllocationScope scope)
576 {
577 void *ptr = vk_alloc(alloc, ma->size, ma->align, scope);
578 if (!ptr)
579 return NULL;
580
581 /* Fill out each of the pointers with their final value.
582 *
583 * for (uint32_t i = 0; i < ma->ptr_count; i++)
584 * *ma->ptrs[i] = ptr + (uintptr_t)*ma->ptrs[i];
585 *
586 * Unfortunately, even though ma->ptr_count is basically guaranteed to be a
587 * constant, GCC is incapable of figuring this out and unrolling the loop
588 * so we have to give it a little help.
589 */
590 STATIC_ASSERT(ARRAY_SIZE(ma->ptrs) == 8);
591 #define _ANV_MULTIALLOC_UPDATE_POINTER(_i) \
592 if ((_i) < ma->ptr_count) \
593 *ma->ptrs[_i] = ptr + (uintptr_t)*ma->ptrs[_i]
594 _ANV_MULTIALLOC_UPDATE_POINTER(0);
595 _ANV_MULTIALLOC_UPDATE_POINTER(1);
596 _ANV_MULTIALLOC_UPDATE_POINTER(2);
597 _ANV_MULTIALLOC_UPDATE_POINTER(3);
598 _ANV_MULTIALLOC_UPDATE_POINTER(4);
599 _ANV_MULTIALLOC_UPDATE_POINTER(5);
600 _ANV_MULTIALLOC_UPDATE_POINTER(6);
601 _ANV_MULTIALLOC_UPDATE_POINTER(7);
602 #undef _ANV_MULTIALLOC_UPDATE_POINTER
603
604 return ptr;
605 }
606
607 __attribute__((always_inline))
608 static inline void *
609 anv_multialloc_alloc2(struct anv_multialloc *ma,
610 const VkAllocationCallbacks *parent_alloc,
611 const VkAllocationCallbacks *alloc,
612 VkSystemAllocationScope scope)
613 {
614 return anv_multialloc_alloc(ma, alloc ? alloc : parent_alloc, scope);
615 }
616
617 struct anv_bo {
618 uint32_t gem_handle;
619
620 uint32_t refcount;
621
622 /* Index into the current validation list. This is used by the
623 * validation list building alrogithm to track which buffers are already
624 * in the validation list so that we can ensure uniqueness.
625 */
626 uint32_t index;
627
628 /* Index for use with util_sparse_array_free_list */
629 uint32_t free_index;
630
631 /* Last known offset. This value is provided by the kernel when we
632 * execbuf and is used as the presumed offset for the next bunch of
633 * relocations.
634 */
635 uint64_t offset;
636
637 uint64_t size;
638
639 /* Map for internally mapped BOs.
640 *
641 * If ANV_BO_WRAPPER is set in flags, map points to the wrapped BO.
642 */
643 void *map;
644
645 /** Flags to pass to the kernel through drm_i915_exec_object2::flags */
646 uint32_t flags;
647
648 /** True if this BO may be shared with other processes */
649 bool is_external:1;
650
651 /** True if this BO is a wrapper
652 *
653 * When set to true, none of the fields in this BO are meaningful except
654 * for anv_bo::is_wrapper and anv_bo::map which points to the actual BO.
655 * See also anv_bo_unwrap(). Wrapper BOs are not allowed when use_softpin
656 * is set in the physical device.
657 */
658 bool is_wrapper:1;
659
660 /** See also ANV_BO_ALLOC_FIXED_ADDRESS */
661 bool has_fixed_address:1;
662
663 /** True if this BO wraps a host pointer */
664 bool from_host_ptr:1;
665 };
666
667 static inline struct anv_bo *
668 anv_bo_unwrap(struct anv_bo *bo)
669 {
670 while (bo->is_wrapper)
671 bo = bo->map;
672 return bo;
673 }
674
675 /* Represents a lock-free linked list of "free" things. This is used by
676 * both the block pool and the state pools. Unfortunately, in order to
677 * solve the ABA problem, we can't use a single uint32_t head.
678 */
679 union anv_free_list {
680 struct {
681 uint32_t offset;
682
683 /* A simple count that is incremented every time the head changes. */
684 uint32_t count;
685 };
686 /* Make sure it's aligned to 64 bits. This will make atomic operations
687 * faster on 32 bit platforms.
688 */
689 uint64_t u64 __attribute__ ((aligned (8)));
690 };
691
692 #define ANV_FREE_LIST_EMPTY ((union anv_free_list) { { UINT32_MAX, 0 } })
693
694 struct anv_block_state {
695 union {
696 struct {
697 uint32_t next;
698 uint32_t end;
699 };
700 /* Make sure it's aligned to 64 bits. This will make atomic operations
701 * faster on 32 bit platforms.
702 */
703 uint64_t u64 __attribute__ ((aligned (8)));
704 };
705 };
706
707 #define anv_block_pool_foreach_bo(bo, pool) \
708 for (struct anv_bo **_pp_bo = (pool)->bos, *bo; \
709 _pp_bo != &(pool)->bos[(pool)->nbos] && (bo = *_pp_bo, true); \
710 _pp_bo++)
711
712 #define ANV_MAX_BLOCK_POOL_BOS 20
713
714 struct anv_block_pool {
715 struct anv_device *device;
716 bool use_softpin;
717
718 /* Wrapper BO for use in relocation lists. This BO is simply a wrapper
719 * around the actual BO so that we grow the pool after the wrapper BO has
720 * been put in a relocation list. This is only used in the non-softpin
721 * case.
722 */
723 struct anv_bo wrapper_bo;
724
725 struct anv_bo *bos[ANV_MAX_BLOCK_POOL_BOS];
726 struct anv_bo *bo;
727 uint32_t nbos;
728
729 uint64_t size;
730
731 /* The address where the start of the pool is pinned. The various bos that
732 * are created as the pool grows will have addresses in the range
733 * [start_address, start_address + BLOCK_POOL_MEMFD_SIZE).
734 */
735 uint64_t start_address;
736
737 /* The offset from the start of the bo to the "center" of the block
738 * pool. Pointers to allocated blocks are given by
739 * bo.map + center_bo_offset + offsets.
740 */
741 uint32_t center_bo_offset;
742
743 /* Current memory map of the block pool. This pointer may or may not
744 * point to the actual beginning of the block pool memory. If
745 * anv_block_pool_alloc_back has ever been called, then this pointer
746 * will point to the "center" position of the buffer and all offsets
747 * (negative or positive) given out by the block pool alloc functions
748 * will be valid relative to this pointer.
749 *
750 * In particular, map == bo.map + center_offset
751 *
752 * DO NOT access this pointer directly. Use anv_block_pool_map() instead,
753 * since it will handle the softpin case as well, where this points to NULL.
754 */
755 void *map;
756 int fd;
757
758 /**
759 * Array of mmaps and gem handles owned by the block pool, reclaimed when
760 * the block pool is destroyed.
761 */
762 struct u_vector mmap_cleanups;
763
764 struct anv_block_state state;
765
766 struct anv_block_state back_state;
767 };
768
769 /* Block pools are backed by a fixed-size 1GB memfd */
770 #define BLOCK_POOL_MEMFD_SIZE (1ul << 30)
771
772 /* The center of the block pool is also the middle of the memfd. This may
773 * change in the future if we decide differently for some reason.
774 */
775 #define BLOCK_POOL_MEMFD_CENTER (BLOCK_POOL_MEMFD_SIZE / 2)
776
777 static inline uint32_t
778 anv_block_pool_size(struct anv_block_pool *pool)
779 {
780 return pool->state.end + pool->back_state.end;
781 }
782
783 struct anv_state {
784 int32_t offset;
785 uint32_t alloc_size;
786 void *map;
787 uint32_t idx;
788 };
789
790 #define ANV_STATE_NULL ((struct anv_state) { .alloc_size = 0 })
791
792 struct anv_fixed_size_state_pool {
793 union anv_free_list free_list;
794 struct anv_block_state block;
795 };
796
797 #define ANV_MIN_STATE_SIZE_LOG2 6
798 #define ANV_MAX_STATE_SIZE_LOG2 21
799
800 #define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2 + 1)
801
802 struct anv_free_entry {
803 uint32_t next;
804 struct anv_state state;
805 };
806
807 struct anv_state_table {
808 struct anv_device *device;
809 int fd;
810 struct anv_free_entry *map;
811 uint32_t size;
812 struct anv_block_state state;
813 struct u_vector cleanups;
814 };
815
816 struct anv_state_pool {
817 struct anv_block_pool block_pool;
818
819 struct anv_state_table table;
820
821 /* The size of blocks which will be allocated from the block pool */
822 uint32_t block_size;
823
824 /** Free list for "back" allocations */
825 union anv_free_list back_alloc_free_list;
826
827 struct anv_fixed_size_state_pool buckets[ANV_STATE_BUCKETS];
828 };
829
830 struct anv_state_stream_block;
831
832 struct anv_state_stream {
833 struct anv_state_pool *state_pool;
834
835 /* The size of blocks to allocate from the state pool */
836 uint32_t block_size;
837
838 /* Current block we're allocating from */
839 struct anv_state block;
840
841 /* Offset into the current block at which to allocate the next state */
842 uint32_t next;
843
844 /* List of all blocks allocated from this pool */
845 struct anv_state_stream_block *block_list;
846 };
847
848 /* The block_pool functions exported for testing only. The block pool should
849 * only be used via a state pool (see below).
850 */
851 VkResult anv_block_pool_init(struct anv_block_pool *pool,
852 struct anv_device *device,
853 uint64_t start_address,
854 uint32_t initial_size);
855 void anv_block_pool_finish(struct anv_block_pool *pool);
856 int32_t anv_block_pool_alloc(struct anv_block_pool *pool,
857 uint32_t block_size, uint32_t *padding);
858 int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool,
859 uint32_t block_size);
860 void* anv_block_pool_map(struct anv_block_pool *pool, int32_t offset);
861
862 VkResult anv_state_pool_init(struct anv_state_pool *pool,
863 struct anv_device *device,
864 uint64_t start_address,
865 uint32_t block_size);
866 void anv_state_pool_finish(struct anv_state_pool *pool);
867 struct anv_state anv_state_pool_alloc(struct anv_state_pool *pool,
868 uint32_t state_size, uint32_t alignment);
869 struct anv_state anv_state_pool_alloc_back(struct anv_state_pool *pool);
870 void anv_state_pool_free(struct anv_state_pool *pool, struct anv_state state);
871 void anv_state_stream_init(struct anv_state_stream *stream,
872 struct anv_state_pool *state_pool,
873 uint32_t block_size);
874 void anv_state_stream_finish(struct anv_state_stream *stream);
875 struct anv_state anv_state_stream_alloc(struct anv_state_stream *stream,
876 uint32_t size, uint32_t alignment);
877
878 VkResult anv_state_table_init(struct anv_state_table *table,
879 struct anv_device *device,
880 uint32_t initial_entries);
881 void anv_state_table_finish(struct anv_state_table *table);
882 VkResult anv_state_table_add(struct anv_state_table *table, uint32_t *idx,
883 uint32_t count);
884 void anv_free_list_push(union anv_free_list *list,
885 struct anv_state_table *table,
886 uint32_t idx, uint32_t count);
887 struct anv_state* anv_free_list_pop(union anv_free_list *list,
888 struct anv_state_table *table);
889
890
891 static inline struct anv_state *
892 anv_state_table_get(struct anv_state_table *table, uint32_t idx)
893 {
894 return &table->map[idx].state;
895 }
896 /**
897 * Implements a pool of re-usable BOs. The interface is identical to that
898 * of block_pool except that each block is its own BO.
899 */
900 struct anv_bo_pool {
901 struct anv_device *device;
902
903 uint64_t bo_flags;
904
905 struct util_sparse_array_free_list free_list[16];
906 };
907
908 void anv_bo_pool_init(struct anv_bo_pool *pool, struct anv_device *device,
909 uint64_t bo_flags);
910 void anv_bo_pool_finish(struct anv_bo_pool *pool);
911 VkResult anv_bo_pool_alloc(struct anv_bo_pool *pool, uint32_t size,
912 struct anv_bo **bo_out);
913 void anv_bo_pool_free(struct anv_bo_pool *pool, struct anv_bo *bo);
914
915 struct anv_scratch_pool {
916 /* Indexed by Per-Thread Scratch Space number (the hardware value) and stage */
917 struct anv_bo *bos[16][MESA_SHADER_STAGES];
918 };
919
920 void anv_scratch_pool_init(struct anv_device *device,
921 struct anv_scratch_pool *pool);
922 void anv_scratch_pool_finish(struct anv_device *device,
923 struct anv_scratch_pool *pool);
924 struct anv_bo *anv_scratch_pool_alloc(struct anv_device *device,
925 struct anv_scratch_pool *pool,
926 gl_shader_stage stage,
927 unsigned per_thread_scratch);
928
929 /** Implements a BO cache that ensures a 1-1 mapping of GEM BOs to anv_bos */
930 struct anv_bo_cache {
931 struct util_sparse_array bo_map;
932 pthread_mutex_t mutex;
933 };
934
935 VkResult anv_bo_cache_init(struct anv_bo_cache *cache);
936 void anv_bo_cache_finish(struct anv_bo_cache *cache);
937
938 struct anv_memory_type {
939 /* Standard bits passed on to the client */
940 VkMemoryPropertyFlags propertyFlags;
941 uint32_t heapIndex;
942
943 /* Driver-internal book-keeping */
944 VkBufferUsageFlags valid_buffer_usage;
945 };
946
947 struct anv_memory_heap {
948 /* Standard bits passed on to the client */
949 VkDeviceSize size;
950 VkMemoryHeapFlags flags;
951
952 /* Driver-internal book-keeping */
953 uint64_t vma_start;
954 uint64_t vma_size;
955 bool supports_48bit_addresses;
956 VkDeviceSize used;
957 };
958
959 struct anv_physical_device {
960 VK_LOADER_DATA _loader_data;
961
962 struct anv_instance * instance;
963 uint32_t chipset_id;
964 bool no_hw;
965 char path[20];
966 const char * name;
967 struct {
968 uint16_t domain;
969 uint8_t bus;
970 uint8_t device;
971 uint8_t function;
972 } pci_info;
973 struct gen_device_info info;
974 /** Amount of "GPU memory" we want to advertise
975 *
976 * Clearly, this value is bogus since Intel is a UMA architecture. On
977 * gen7 platforms, we are limited by GTT size unless we want to implement
978 * fine-grained tracking and GTT splitting. On Broadwell and above we are
979 * practically unlimited. However, we will never report more than 3/4 of
980 * the total system ram to try and avoid running out of RAM.
981 */
982 bool supports_48bit_addresses;
983 struct brw_compiler * compiler;
984 struct isl_device isl_dev;
985 struct gen_perf_config * perf;
986 int cmd_parser_version;
987 bool has_exec_async;
988 bool has_exec_capture;
989 bool has_exec_fence;
990 bool has_syncobj;
991 bool has_syncobj_wait;
992 bool has_context_priority;
993 bool use_softpin;
994 bool has_context_isolation;
995 bool has_mem_available;
996 bool always_use_bindless;
997
998 /** True if we can access buffers using A64 messages */
999 bool has_a64_buffer_access;
1000 /** True if we can use bindless access for images */
1001 bool has_bindless_images;
1002 /** True if we can use bindless access for samplers */
1003 bool has_bindless_samplers;
1004
1005 struct anv_device_extension_table supported_extensions;
1006 struct anv_physical_device_dispatch_table dispatch;
1007
1008 uint32_t eu_total;
1009 uint32_t subslice_total;
1010
1011 struct {
1012 uint32_t type_count;
1013 struct anv_memory_type types[VK_MAX_MEMORY_TYPES];
1014 uint32_t heap_count;
1015 struct anv_memory_heap heaps[VK_MAX_MEMORY_HEAPS];
1016 } memory;
1017
1018 uint8_t driver_build_sha1[20];
1019 uint8_t pipeline_cache_uuid[VK_UUID_SIZE];
1020 uint8_t driver_uuid[VK_UUID_SIZE];
1021 uint8_t device_uuid[VK_UUID_SIZE];
1022
1023 struct disk_cache * disk_cache;
1024
1025 struct wsi_device wsi_device;
1026 int local_fd;
1027 int master_fd;
1028 };
1029
1030 struct anv_app_info {
1031 const char* app_name;
1032 uint32_t app_version;
1033 const char* engine_name;
1034 uint32_t engine_version;
1035 uint32_t api_version;
1036 };
1037
1038 struct anv_instance {
1039 VK_LOADER_DATA _loader_data;
1040
1041 VkAllocationCallbacks alloc;
1042
1043 struct anv_app_info app_info;
1044
1045 struct anv_instance_extension_table enabled_extensions;
1046 struct anv_instance_dispatch_table dispatch;
1047 struct anv_device_dispatch_table device_dispatch;
1048
1049 int physicalDeviceCount;
1050 struct anv_physical_device physicalDevice;
1051
1052 bool pipeline_cache_enabled;
1053
1054 struct vk_debug_report_instance debug_report_callbacks;
1055
1056 struct driOptionCache dri_options;
1057 struct driOptionCache available_dri_options;
1058 };
1059
1060 VkResult anv_init_wsi(struct anv_physical_device *physical_device);
1061 void anv_finish_wsi(struct anv_physical_device *physical_device);
1062
1063 uint32_t anv_physical_device_api_version(struct anv_physical_device *dev);
1064 bool anv_physical_device_extension_supported(struct anv_physical_device *dev,
1065 const char *name);
1066
1067 struct anv_queue_submit {
1068 struct anv_cmd_buffer * cmd_buffer;
1069
1070 uint32_t fence_count;
1071 uint32_t fence_array_length;
1072 struct drm_i915_gem_exec_fence * fences;
1073
1074 uint32_t temporary_semaphore_count;
1075 uint32_t temporary_semaphore_array_length;
1076 struct anv_semaphore_impl * temporary_semaphores;
1077
1078 /* Semaphores to be signaled with a SYNC_FD. */
1079 struct anv_semaphore ** sync_fd_semaphores;
1080 uint32_t sync_fd_semaphore_count;
1081 uint32_t sync_fd_semaphore_array_length;
1082
1083 /* Allocated only with non shareable timelines. */
1084 struct anv_timeline ** wait_timelines;
1085 uint32_t wait_timeline_count;
1086 uint32_t wait_timeline_array_length;
1087 uint64_t * wait_timeline_values;
1088
1089 struct anv_timeline ** signal_timelines;
1090 uint32_t signal_timeline_count;
1091 uint32_t signal_timeline_array_length;
1092 uint64_t * signal_timeline_values;
1093
1094 int in_fence;
1095 bool need_out_fence;
1096 int out_fence;
1097
1098 uint32_t fence_bo_count;
1099 uint32_t fence_bo_array_length;
1100 /* An array of struct anv_bo pointers with lower bit used as a flag to
1101 * signal we will wait on that BO (see anv_(un)pack_ptr).
1102 */
1103 uintptr_t * fence_bos;
1104
1105 const VkAllocationCallbacks * alloc;
1106 VkSystemAllocationScope alloc_scope;
1107
1108 struct anv_bo * simple_bo;
1109 uint32_t simple_bo_size;
1110
1111 struct list_head link;
1112 };
1113
1114 struct anv_queue {
1115 VK_LOADER_DATA _loader_data;
1116
1117 struct anv_device * device;
1118
1119 /*
1120 * A list of struct anv_queue_submit to be submitted to i915.
1121 */
1122 struct list_head queued_submits;
1123
1124 VkDeviceQueueCreateFlags flags;
1125 };
1126
1127 struct anv_pipeline_cache {
1128 struct anv_device * device;
1129 pthread_mutex_t mutex;
1130
1131 struct hash_table * nir_cache;
1132
1133 struct hash_table * cache;
1134 };
1135
1136 struct nir_xfb_info;
1137 struct anv_pipeline_bind_map;
1138
1139 void anv_pipeline_cache_init(struct anv_pipeline_cache *cache,
1140 struct anv_device *device,
1141 bool cache_enabled);
1142 void anv_pipeline_cache_finish(struct anv_pipeline_cache *cache);
1143
1144 struct anv_shader_bin *
1145 anv_pipeline_cache_search(struct anv_pipeline_cache *cache,
1146 const void *key, uint32_t key_size);
1147 struct anv_shader_bin *
1148 anv_pipeline_cache_upload_kernel(struct anv_pipeline_cache *cache,
1149 const void *key_data, uint32_t key_size,
1150 const void *kernel_data, uint32_t kernel_size,
1151 const void *constant_data,
1152 uint32_t constant_data_size,
1153 const struct brw_stage_prog_data *prog_data,
1154 uint32_t prog_data_size,
1155 const struct brw_compile_stats *stats,
1156 uint32_t num_stats,
1157 const struct nir_xfb_info *xfb_info,
1158 const struct anv_pipeline_bind_map *bind_map);
1159
1160 struct anv_shader_bin *
1161 anv_device_search_for_kernel(struct anv_device *device,
1162 struct anv_pipeline_cache *cache,
1163 const void *key_data, uint32_t key_size,
1164 bool *user_cache_bit);
1165
1166 struct anv_shader_bin *
1167 anv_device_upload_kernel(struct anv_device *device,
1168 struct anv_pipeline_cache *cache,
1169 const void *key_data, uint32_t key_size,
1170 const void *kernel_data, uint32_t kernel_size,
1171 const void *constant_data,
1172 uint32_t constant_data_size,
1173 const struct brw_stage_prog_data *prog_data,
1174 uint32_t prog_data_size,
1175 const struct brw_compile_stats *stats,
1176 uint32_t num_stats,
1177 const struct nir_xfb_info *xfb_info,
1178 const struct anv_pipeline_bind_map *bind_map);
1179
1180 struct nir_shader;
1181 struct nir_shader_compiler_options;
1182
1183 struct nir_shader *
1184 anv_device_search_for_nir(struct anv_device *device,
1185 struct anv_pipeline_cache *cache,
1186 const struct nir_shader_compiler_options *nir_options,
1187 unsigned char sha1_key[20],
1188 void *mem_ctx);
1189
1190 void
1191 anv_device_upload_nir(struct anv_device *device,
1192 struct anv_pipeline_cache *cache,
1193 const struct nir_shader *nir,
1194 unsigned char sha1_key[20]);
1195
1196 struct anv_device {
1197 VK_LOADER_DATA _loader_data;
1198
1199 VkAllocationCallbacks alloc;
1200
1201 struct anv_instance * instance;
1202 uint32_t chipset_id;
1203 bool no_hw;
1204 struct gen_device_info info;
1205 struct isl_device isl_dev;
1206 int context_id;
1207 int fd;
1208 bool can_chain_batches;
1209 bool robust_buffer_access;
1210 struct anv_device_extension_table enabled_extensions;
1211 struct anv_device_dispatch_table dispatch;
1212
1213 pthread_mutex_t vma_mutex;
1214 struct util_vma_heap vma_lo;
1215 struct util_vma_heap vma_hi;
1216 uint64_t vma_lo_available;
1217 uint64_t vma_hi_available;
1218
1219 /** List of all anv_device_memory objects */
1220 struct list_head memory_objects;
1221
1222 struct anv_bo_pool batch_bo_pool;
1223
1224 struct anv_bo_cache bo_cache;
1225
1226 struct anv_state_pool dynamic_state_pool;
1227 struct anv_state_pool instruction_state_pool;
1228 struct anv_state_pool binding_table_pool;
1229 struct anv_state_pool surface_state_pool;
1230
1231 struct anv_bo * workaround_bo;
1232 struct anv_bo * trivial_batch_bo;
1233 struct anv_bo * hiz_clear_bo;
1234
1235 struct anv_pipeline_cache default_pipeline_cache;
1236 struct blorp_context blorp;
1237
1238 struct anv_state border_colors;
1239
1240 struct anv_state slice_hash;
1241
1242 struct anv_queue queue;
1243
1244 struct anv_scratch_pool scratch_pool;
1245
1246 pthread_mutex_t mutex;
1247 pthread_cond_t queue_submit;
1248 int _lost;
1249
1250 struct gen_batch_decode_ctx decoder_ctx;
1251 /*
1252 * When decoding a anv_cmd_buffer, we might need to search for BOs through
1253 * the cmd_buffer's list.
1254 */
1255 struct anv_cmd_buffer *cmd_buffer_being_decoded;
1256
1257 int perf_fd; /* -1 if no opened */
1258 uint64_t perf_metric; /* 0 if unset */
1259
1260 struct gen_aux_map_context *aux_map_ctx;
1261 };
1262
1263 static inline struct anv_state_pool *
1264 anv_binding_table_pool(struct anv_device *device)
1265 {
1266 if (device->instance->physicalDevice.use_softpin)
1267 return &device->binding_table_pool;
1268 else
1269 return &device->surface_state_pool;
1270 }
1271
1272 static inline struct anv_state
1273 anv_binding_table_pool_alloc(struct anv_device *device) {
1274 if (device->instance->physicalDevice.use_softpin)
1275 return anv_state_pool_alloc(&device->binding_table_pool,
1276 device->binding_table_pool.block_size, 0);
1277 else
1278 return anv_state_pool_alloc_back(&device->surface_state_pool);
1279 }
1280
1281 static inline void
1282 anv_binding_table_pool_free(struct anv_device *device, struct anv_state state) {
1283 anv_state_pool_free(anv_binding_table_pool(device), state);
1284 }
1285
1286 static inline uint32_t
1287 anv_mocs_for_bo(const struct anv_device *device, const struct anv_bo *bo)
1288 {
1289 if (bo->is_external)
1290 return device->isl_dev.mocs.external;
1291 else
1292 return device->isl_dev.mocs.internal;
1293 }
1294
1295 void anv_device_init_blorp(struct anv_device *device);
1296 void anv_device_finish_blorp(struct anv_device *device);
1297
1298 void _anv_device_set_all_queue_lost(struct anv_device *device);
1299 VkResult _anv_device_set_lost(struct anv_device *device,
1300 const char *file, int line,
1301 const char *msg, ...)
1302 anv_printflike(4, 5);
1303 VkResult _anv_queue_set_lost(struct anv_queue *queue,
1304 const char *file, int line,
1305 const char *msg, ...)
1306 anv_printflike(4, 5);
1307 #define anv_device_set_lost(dev, ...) \
1308 _anv_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
1309 #define anv_queue_set_lost(queue, ...) \
1310 _anv_queue_set_lost(queue, __FILE__, __LINE__, __VA_ARGS__)
1311
1312 static inline bool
1313 anv_device_is_lost(struct anv_device *device)
1314 {
1315 return unlikely(p_atomic_read(&device->_lost));
1316 }
1317
1318 VkResult anv_device_query_status(struct anv_device *device);
1319
1320
1321 enum anv_bo_alloc_flags {
1322 /** Specifies that the BO must have a 32-bit address
1323 *
1324 * This is the opposite of EXEC_OBJECT_SUPPORTS_48B_ADDRESS.
1325 */
1326 ANV_BO_ALLOC_32BIT_ADDRESS = (1 << 0),
1327
1328 /** Specifies that the BO may be shared externally */
1329 ANV_BO_ALLOC_EXTERNAL = (1 << 1),
1330
1331 /** Specifies that the BO should be mapped */
1332 ANV_BO_ALLOC_MAPPED = (1 << 2),
1333
1334 /** Specifies that the BO should be snooped so we get coherency */
1335 ANV_BO_ALLOC_SNOOPED = (1 << 3),
1336
1337 /** Specifies that the BO should be captured in error states */
1338 ANV_BO_ALLOC_CAPTURE = (1 << 4),
1339
1340 /** Specifies that the BO will have an address assigned by the caller */
1341 ANV_BO_ALLOC_FIXED_ADDRESS = (1 << 5),
1342
1343 /** Enables implicit synchronization on the BO
1344 *
1345 * This is the opposite of EXEC_OBJECT_ASYNC.
1346 */
1347 ANV_BO_ALLOC_IMPLICIT_SYNC = (1 << 6),
1348
1349 /** Enables implicit synchronization on the BO
1350 *
1351 * This is equivalent to EXEC_OBJECT_WRITE.
1352 */
1353 ANV_BO_ALLOC_IMPLICIT_WRITE = (1 << 7),
1354 };
1355
1356 VkResult anv_device_alloc_bo(struct anv_device *device, uint64_t size,
1357 enum anv_bo_alloc_flags alloc_flags,
1358 struct anv_bo **bo);
1359 VkResult anv_device_import_bo_from_host_ptr(struct anv_device *device,
1360 void *host_ptr, uint32_t size,
1361 enum anv_bo_alloc_flags alloc_flags,
1362 struct anv_bo **bo_out);
1363 VkResult anv_device_import_bo(struct anv_device *device, int fd,
1364 enum anv_bo_alloc_flags alloc_flags,
1365 struct anv_bo **bo);
1366 VkResult anv_device_export_bo(struct anv_device *device,
1367 struct anv_bo *bo, int *fd_out);
1368 void anv_device_release_bo(struct anv_device *device,
1369 struct anv_bo *bo);
1370
1371 static inline struct anv_bo *
1372 anv_device_lookup_bo(struct anv_device *device, uint32_t gem_handle)
1373 {
1374 return util_sparse_array_get(&device->bo_cache.bo_map, gem_handle);
1375 }
1376
1377 VkResult anv_device_bo_busy(struct anv_device *device, struct anv_bo *bo);
1378 VkResult anv_device_wait(struct anv_device *device, struct anv_bo *bo,
1379 int64_t timeout);
1380
1381 VkResult anv_queue_init(struct anv_device *device, struct anv_queue *queue);
1382 void anv_queue_finish(struct anv_queue *queue);
1383
1384 VkResult anv_queue_execbuf_locked(struct anv_queue *queue, struct anv_queue_submit *submit);
1385 VkResult anv_queue_submit_simple_batch(struct anv_queue *queue,
1386 struct anv_batch *batch);
1387
1388 uint64_t anv_gettime_ns(void);
1389 uint64_t anv_get_absolute_timeout(uint64_t timeout);
1390
1391 void* anv_gem_mmap(struct anv_device *device,
1392 uint32_t gem_handle, uint64_t offset, uint64_t size, uint32_t flags);
1393 void anv_gem_munmap(void *p, uint64_t size);
1394 uint32_t anv_gem_create(struct anv_device *device, uint64_t size);
1395 void anv_gem_close(struct anv_device *device, uint32_t gem_handle);
1396 uint32_t anv_gem_userptr(struct anv_device *device, void *mem, size_t size);
1397 int anv_gem_busy(struct anv_device *device, uint32_t gem_handle);
1398 int anv_gem_wait(struct anv_device *device, uint32_t gem_handle, int64_t *timeout_ns);
1399 int anv_gem_execbuffer(struct anv_device *device,
1400 struct drm_i915_gem_execbuffer2 *execbuf);
1401 int anv_gem_set_tiling(struct anv_device *device, uint32_t gem_handle,
1402 uint32_t stride, uint32_t tiling);
1403 int anv_gem_create_context(struct anv_device *device);
1404 bool anv_gem_has_context_priority(int fd);
1405 int anv_gem_destroy_context(struct anv_device *device, int context);
1406 int anv_gem_set_context_param(int fd, int context, uint32_t param,
1407 uint64_t value);
1408 int anv_gem_get_context_param(int fd, int context, uint32_t param,
1409 uint64_t *value);
1410 int anv_gem_get_param(int fd, uint32_t param);
1411 int anv_gem_get_tiling(struct anv_device *device, uint32_t gem_handle);
1412 bool anv_gem_get_bit6_swizzle(int fd, uint32_t tiling);
1413 int anv_gem_get_aperture(int fd, uint64_t *size);
1414 int anv_gem_gpu_get_reset_stats(struct anv_device *device,
1415 uint32_t *active, uint32_t *pending);
1416 int anv_gem_handle_to_fd(struct anv_device *device, uint32_t gem_handle);
1417 int anv_gem_reg_read(struct anv_device *device,
1418 uint32_t offset, uint64_t *result);
1419 uint32_t anv_gem_fd_to_handle(struct anv_device *device, int fd);
1420 int anv_gem_set_caching(struct anv_device *device, uint32_t gem_handle, uint32_t caching);
1421 int anv_gem_set_domain(struct anv_device *device, uint32_t gem_handle,
1422 uint32_t read_domains, uint32_t write_domain);
1423 int anv_gem_sync_file_merge(struct anv_device *device, int fd1, int fd2);
1424 uint32_t anv_gem_syncobj_create(struct anv_device *device, uint32_t flags);
1425 void anv_gem_syncobj_destroy(struct anv_device *device, uint32_t handle);
1426 int anv_gem_syncobj_handle_to_fd(struct anv_device *device, uint32_t handle);
1427 uint32_t anv_gem_syncobj_fd_to_handle(struct anv_device *device, int fd);
1428 int anv_gem_syncobj_export_sync_file(struct anv_device *device,
1429 uint32_t handle);
1430 int anv_gem_syncobj_import_sync_file(struct anv_device *device,
1431 uint32_t handle, int fd);
1432 void anv_gem_syncobj_reset(struct anv_device *device, uint32_t handle);
1433 bool anv_gem_supports_syncobj_wait(int fd);
1434 int anv_gem_syncobj_wait(struct anv_device *device,
1435 uint32_t *handles, uint32_t num_handles,
1436 int64_t abs_timeout_ns, bool wait_all);
1437
1438 bool anv_vma_alloc(struct anv_device *device, struct anv_bo *bo);
1439 void anv_vma_free(struct anv_device *device, struct anv_bo *bo);
1440
1441 struct anv_reloc_list {
1442 uint32_t num_relocs;
1443 uint32_t array_length;
1444 struct drm_i915_gem_relocation_entry * relocs;
1445 struct anv_bo ** reloc_bos;
1446 uint32_t dep_words;
1447 BITSET_WORD * deps;
1448 };
1449
1450 VkResult anv_reloc_list_init(struct anv_reloc_list *list,
1451 const VkAllocationCallbacks *alloc);
1452 void anv_reloc_list_finish(struct anv_reloc_list *list,
1453 const VkAllocationCallbacks *alloc);
1454
1455 VkResult anv_reloc_list_add(struct anv_reloc_list *list,
1456 const VkAllocationCallbacks *alloc,
1457 uint32_t offset, struct anv_bo *target_bo,
1458 uint32_t delta, uint64_t *address_u64_out);
1459
1460 struct anv_batch_bo {
1461 /* Link in the anv_cmd_buffer.owned_batch_bos list */
1462 struct list_head link;
1463
1464 struct anv_bo * bo;
1465
1466 /* Bytes actually consumed in this batch BO */
1467 uint32_t length;
1468
1469 struct anv_reloc_list relocs;
1470 };
1471
1472 struct anv_batch {
1473 const VkAllocationCallbacks * alloc;
1474
1475 void * start;
1476 void * end;
1477 void * next;
1478
1479 struct anv_reloc_list * relocs;
1480
1481 /* This callback is called (with the associated user data) in the event
1482 * that the batch runs out of space.
1483 */
1484 VkResult (*extend_cb)(struct anv_batch *, void *);
1485 void * user_data;
1486
1487 /**
1488 * Current error status of the command buffer. Used to track inconsistent
1489 * or incomplete command buffer states that are the consequence of run-time
1490 * errors such as out of memory scenarios. We want to track this in the
1491 * batch because the command buffer object is not visible to some parts
1492 * of the driver.
1493 */
1494 VkResult status;
1495 };
1496
1497 void *anv_batch_emit_dwords(struct anv_batch *batch, int num_dwords);
1498 void anv_batch_emit_batch(struct anv_batch *batch, struct anv_batch *other);
1499 uint64_t anv_batch_emit_reloc(struct anv_batch *batch,
1500 void *location, struct anv_bo *bo, uint32_t offset);
1501
1502 static inline VkResult
1503 anv_batch_set_error(struct anv_batch *batch, VkResult error)
1504 {
1505 assert(error != VK_SUCCESS);
1506 if (batch->status == VK_SUCCESS)
1507 batch->status = error;
1508 return batch->status;
1509 }
1510
1511 static inline bool
1512 anv_batch_has_error(struct anv_batch *batch)
1513 {
1514 return batch->status != VK_SUCCESS;
1515 }
1516
1517 struct anv_address {
1518 struct anv_bo *bo;
1519 uint32_t offset;
1520 };
1521
1522 #define ANV_NULL_ADDRESS ((struct anv_address) { NULL, 0 })
1523
1524 static inline bool
1525 anv_address_is_null(struct anv_address addr)
1526 {
1527 return addr.bo == NULL && addr.offset == 0;
1528 }
1529
1530 static inline uint64_t
1531 anv_address_physical(struct anv_address addr)
1532 {
1533 if (addr.bo && (addr.bo->flags & EXEC_OBJECT_PINNED))
1534 return gen_canonical_address(addr.bo->offset + addr.offset);
1535 else
1536 return gen_canonical_address(addr.offset);
1537 }
1538
1539 static inline struct anv_address
1540 anv_address_add(struct anv_address addr, uint64_t offset)
1541 {
1542 addr.offset += offset;
1543 return addr;
1544 }
1545
1546 static inline void
1547 write_reloc(const struct anv_device *device, void *p, uint64_t v, bool flush)
1548 {
1549 unsigned reloc_size = 0;
1550 if (device->info.gen >= 8) {
1551 reloc_size = sizeof(uint64_t);
1552 *(uint64_t *)p = gen_canonical_address(v);
1553 } else {
1554 reloc_size = sizeof(uint32_t);
1555 *(uint32_t *)p = v;
1556 }
1557
1558 if (flush && !device->info.has_llc)
1559 gen_flush_range(p, reloc_size);
1560 }
1561
1562 static inline uint64_t
1563 _anv_combine_address(struct anv_batch *batch, void *location,
1564 const struct anv_address address, uint32_t delta)
1565 {
1566 if (address.bo == NULL) {
1567 return address.offset + delta;
1568 } else {
1569 assert(batch->start <= location && location < batch->end);
1570
1571 return anv_batch_emit_reloc(batch, location, address.bo, address.offset + delta);
1572 }
1573 }
1574
1575 #define __gen_address_type struct anv_address
1576 #define __gen_user_data struct anv_batch
1577 #define __gen_combine_address _anv_combine_address
1578
1579 /* Wrapper macros needed to work around preprocessor argument issues. In
1580 * particular, arguments don't get pre-evaluated if they are concatenated.
1581 * This means that, if you pass GENX(3DSTATE_PS) into the emit macro, the
1582 * GENX macro won't get evaluated if the emit macro contains "cmd ## foo".
1583 * We can work around this easily enough with these helpers.
1584 */
1585 #define __anv_cmd_length(cmd) cmd ## _length
1586 #define __anv_cmd_length_bias(cmd) cmd ## _length_bias
1587 #define __anv_cmd_header(cmd) cmd ## _header
1588 #define __anv_cmd_pack(cmd) cmd ## _pack
1589 #define __anv_reg_num(reg) reg ## _num
1590
1591 #define anv_pack_struct(dst, struc, ...) do { \
1592 struct struc __template = { \
1593 __VA_ARGS__ \
1594 }; \
1595 __anv_cmd_pack(struc)(NULL, dst, &__template); \
1596 VG(VALGRIND_CHECK_MEM_IS_DEFINED(dst, __anv_cmd_length(struc) * 4)); \
1597 } while (0)
1598
1599 #define anv_batch_emitn(batch, n, cmd, ...) ({ \
1600 void *__dst = anv_batch_emit_dwords(batch, n); \
1601 if (__dst) { \
1602 struct cmd __template = { \
1603 __anv_cmd_header(cmd), \
1604 .DWordLength = n - __anv_cmd_length_bias(cmd), \
1605 __VA_ARGS__ \
1606 }; \
1607 __anv_cmd_pack(cmd)(batch, __dst, &__template); \
1608 } \
1609 __dst; \
1610 })
1611
1612 #define anv_batch_emit_merge(batch, dwords0, dwords1) \
1613 do { \
1614 uint32_t *dw; \
1615 \
1616 STATIC_ASSERT(ARRAY_SIZE(dwords0) == ARRAY_SIZE(dwords1)); \
1617 dw = anv_batch_emit_dwords((batch), ARRAY_SIZE(dwords0)); \
1618 if (!dw) \
1619 break; \
1620 for (uint32_t i = 0; i < ARRAY_SIZE(dwords0); i++) \
1621 dw[i] = (dwords0)[i] | (dwords1)[i]; \
1622 VG(VALGRIND_CHECK_MEM_IS_DEFINED(dw, ARRAY_SIZE(dwords0) * 4));\
1623 } while (0)
1624
1625 #define anv_batch_emit(batch, cmd, name) \
1626 for (struct cmd name = { __anv_cmd_header(cmd) }, \
1627 *_dst = anv_batch_emit_dwords(batch, __anv_cmd_length(cmd)); \
1628 __builtin_expect(_dst != NULL, 1); \
1629 ({ __anv_cmd_pack(cmd)(batch, _dst, &name); \
1630 VG(VALGRIND_CHECK_MEM_IS_DEFINED(_dst, __anv_cmd_length(cmd) * 4)); \
1631 _dst = NULL; \
1632 }))
1633
1634 struct anv_device_memory {
1635 struct list_head link;
1636
1637 struct anv_bo * bo;
1638 struct anv_memory_type * type;
1639 VkDeviceSize map_size;
1640 void * map;
1641
1642 /* If set, we are holding reference to AHardwareBuffer
1643 * which we must release when memory is freed.
1644 */
1645 struct AHardwareBuffer * ahw;
1646
1647 /* If set, this memory comes from a host pointer. */
1648 void * host_ptr;
1649 };
1650
1651 /**
1652 * Header for Vertex URB Entry (VUE)
1653 */
1654 struct anv_vue_header {
1655 uint32_t Reserved;
1656 uint32_t RTAIndex; /* RenderTargetArrayIndex */
1657 uint32_t ViewportIndex;
1658 float PointWidth;
1659 };
1660
1661 /** Struct representing a sampled image descriptor
1662 *
1663 * This descriptor layout is used for sampled images, bare sampler, and
1664 * combined image/sampler descriptors.
1665 */
1666 struct anv_sampled_image_descriptor {
1667 /** Bindless image handle
1668 *
1669 * This is expected to already be shifted such that the 20-bit
1670 * SURFACE_STATE table index is in the top 20 bits.
1671 */
1672 uint32_t image;
1673
1674 /** Bindless sampler handle
1675 *
1676 * This is assumed to be a 32B-aligned SAMPLER_STATE pointer relative
1677 * to the dynamic state base address.
1678 */
1679 uint32_t sampler;
1680 };
1681
1682 struct anv_texture_swizzle_descriptor {
1683 /** Texture swizzle
1684 *
1685 * See also nir_intrinsic_channel_select_intel
1686 */
1687 uint8_t swizzle[4];
1688
1689 /** Unused padding to ensure the struct is a multiple of 64 bits */
1690 uint32_t _pad;
1691 };
1692
1693 /** Struct representing a storage image descriptor */
1694 struct anv_storage_image_descriptor {
1695 /** Bindless image handles
1696 *
1697 * These are expected to already be shifted such that the 20-bit
1698 * SURFACE_STATE table index is in the top 20 bits.
1699 */
1700 uint32_t read_write;
1701 uint32_t write_only;
1702 };
1703
1704 /** Struct representing a address/range descriptor
1705 *
1706 * The fields of this struct correspond directly to the data layout of
1707 * nir_address_format_64bit_bounded_global addresses. The last field is the
1708 * offset in the NIR address so it must be zero so that when you load the
1709 * descriptor you get a pointer to the start of the range.
1710 */
1711 struct anv_address_range_descriptor {
1712 uint64_t address;
1713 uint32_t range;
1714 uint32_t zero;
1715 };
1716
1717 enum anv_descriptor_data {
1718 /** The descriptor contains a BTI reference to a surface state */
1719 ANV_DESCRIPTOR_SURFACE_STATE = (1 << 0),
1720 /** The descriptor contains a BTI reference to a sampler state */
1721 ANV_DESCRIPTOR_SAMPLER_STATE = (1 << 1),
1722 /** The descriptor contains an actual buffer view */
1723 ANV_DESCRIPTOR_BUFFER_VIEW = (1 << 2),
1724 /** The descriptor contains auxiliary image layout data */
1725 ANV_DESCRIPTOR_IMAGE_PARAM = (1 << 3),
1726 /** The descriptor contains auxiliary image layout data */
1727 ANV_DESCRIPTOR_INLINE_UNIFORM = (1 << 4),
1728 /** anv_address_range_descriptor with a buffer address and range */
1729 ANV_DESCRIPTOR_ADDRESS_RANGE = (1 << 5),
1730 /** Bindless surface handle */
1731 ANV_DESCRIPTOR_SAMPLED_IMAGE = (1 << 6),
1732 /** Storage image handles */
1733 ANV_DESCRIPTOR_STORAGE_IMAGE = (1 << 7),
1734 /** Storage image handles */
1735 ANV_DESCRIPTOR_TEXTURE_SWIZZLE = (1 << 8),
1736 };
1737
1738 struct anv_descriptor_set_binding_layout {
1739 #ifndef NDEBUG
1740 /* The type of the descriptors in this binding */
1741 VkDescriptorType type;
1742 #endif
1743
1744 /* Flags provided when this binding was created */
1745 VkDescriptorBindingFlagsEXT flags;
1746
1747 /* Bitfield representing the type of data this descriptor contains */
1748 enum anv_descriptor_data data;
1749
1750 /* Maximum number of YCbCr texture/sampler planes */
1751 uint8_t max_plane_count;
1752
1753 /* Number of array elements in this binding (or size in bytes for inline
1754 * uniform data)
1755 */
1756 uint16_t array_size;
1757
1758 /* Index into the flattend descriptor set */
1759 uint16_t descriptor_index;
1760
1761 /* Index into the dynamic state array for a dynamic buffer */
1762 int16_t dynamic_offset_index;
1763
1764 /* Index into the descriptor set buffer views */
1765 int16_t buffer_view_index;
1766
1767 /* Offset into the descriptor buffer where this descriptor lives */
1768 uint32_t descriptor_offset;
1769
1770 /* Immutable samplers (or NULL if no immutable samplers) */
1771 struct anv_sampler **immutable_samplers;
1772 };
1773
1774 unsigned anv_descriptor_size(const struct anv_descriptor_set_binding_layout *layout);
1775
1776 unsigned anv_descriptor_type_size(const struct anv_physical_device *pdevice,
1777 VkDescriptorType type);
1778
1779 bool anv_descriptor_supports_bindless(const struct anv_physical_device *pdevice,
1780 const struct anv_descriptor_set_binding_layout *binding,
1781 bool sampler);
1782
1783 bool anv_descriptor_requires_bindless(const struct anv_physical_device *pdevice,
1784 const struct anv_descriptor_set_binding_layout *binding,
1785 bool sampler);
1786
1787 struct anv_descriptor_set_layout {
1788 /* Descriptor set layouts can be destroyed at almost any time */
1789 uint32_t ref_cnt;
1790
1791 /* Number of bindings in this descriptor set */
1792 uint16_t binding_count;
1793
1794 /* Total size of the descriptor set with room for all array entries */
1795 uint16_t size;
1796
1797 /* Shader stages affected by this descriptor set */
1798 uint16_t shader_stages;
1799
1800 /* Number of buffer views in this descriptor set */
1801 uint16_t buffer_view_count;
1802
1803 /* Number of dynamic offsets used by this descriptor set */
1804 uint16_t dynamic_offset_count;
1805
1806 /* Size of the descriptor buffer for this descriptor set */
1807 uint32_t descriptor_buffer_size;
1808
1809 /* Bindings in this descriptor set */
1810 struct anv_descriptor_set_binding_layout binding[0];
1811 };
1812
1813 static inline void
1814 anv_descriptor_set_layout_ref(struct anv_descriptor_set_layout *layout)
1815 {
1816 assert(layout && layout->ref_cnt >= 1);
1817 p_atomic_inc(&layout->ref_cnt);
1818 }
1819
1820 static inline void
1821 anv_descriptor_set_layout_unref(struct anv_device *device,
1822 struct anv_descriptor_set_layout *layout)
1823 {
1824 assert(layout && layout->ref_cnt >= 1);
1825 if (p_atomic_dec_zero(&layout->ref_cnt))
1826 vk_free(&device->alloc, layout);
1827 }
1828
1829 struct anv_descriptor {
1830 VkDescriptorType type;
1831
1832 union {
1833 struct {
1834 VkImageLayout layout;
1835 struct anv_image_view *image_view;
1836 struct anv_sampler *sampler;
1837 };
1838
1839 struct {
1840 struct anv_buffer *buffer;
1841 uint64_t offset;
1842 uint64_t range;
1843 };
1844
1845 struct anv_buffer_view *buffer_view;
1846 };
1847 };
1848
1849 struct anv_descriptor_set {
1850 struct anv_descriptor_pool *pool;
1851 struct anv_descriptor_set_layout *layout;
1852 uint32_t size;
1853
1854 /* State relative to anv_descriptor_pool::bo */
1855 struct anv_state desc_mem;
1856 /* Surface state for the descriptor buffer */
1857 struct anv_state desc_surface_state;
1858
1859 uint32_t buffer_view_count;
1860 struct anv_buffer_view *buffer_views;
1861
1862 /* Link to descriptor pool's desc_sets list . */
1863 struct list_head pool_link;
1864
1865 struct anv_descriptor descriptors[0];
1866 };
1867
1868 struct anv_buffer_view {
1869 enum isl_format format; /**< VkBufferViewCreateInfo::format */
1870 uint64_t range; /**< VkBufferViewCreateInfo::range */
1871
1872 struct anv_address address;
1873
1874 struct anv_state surface_state;
1875 struct anv_state storage_surface_state;
1876 struct anv_state writeonly_storage_surface_state;
1877
1878 struct brw_image_param storage_image_param;
1879 };
1880
1881 struct anv_push_descriptor_set {
1882 struct anv_descriptor_set set;
1883
1884 /* Put this field right behind anv_descriptor_set so it fills up the
1885 * descriptors[0] field. */
1886 struct anv_descriptor descriptors[MAX_PUSH_DESCRIPTORS];
1887
1888 /** True if the descriptor set buffer has been referenced by a draw or
1889 * dispatch command.
1890 */
1891 bool set_used_on_gpu;
1892
1893 struct anv_buffer_view buffer_views[MAX_PUSH_DESCRIPTORS];
1894 };
1895
1896 struct anv_descriptor_pool {
1897 uint32_t size;
1898 uint32_t next;
1899 uint32_t free_list;
1900
1901 struct anv_bo *bo;
1902 struct util_vma_heap bo_heap;
1903
1904 struct anv_state_stream surface_state_stream;
1905 void *surface_state_free_list;
1906
1907 struct list_head desc_sets;
1908
1909 char data[0];
1910 };
1911
1912 enum anv_descriptor_template_entry_type {
1913 ANV_DESCRIPTOR_TEMPLATE_ENTRY_TYPE_IMAGE,
1914 ANV_DESCRIPTOR_TEMPLATE_ENTRY_TYPE_BUFFER,
1915 ANV_DESCRIPTOR_TEMPLATE_ENTRY_TYPE_BUFFER_VIEW
1916 };
1917
1918 struct anv_descriptor_template_entry {
1919 /* The type of descriptor in this entry */
1920 VkDescriptorType type;
1921
1922 /* Binding in the descriptor set */
1923 uint32_t binding;
1924
1925 /* Offset at which to write into the descriptor set binding */
1926 uint32_t array_element;
1927
1928 /* Number of elements to write into the descriptor set binding */
1929 uint32_t array_count;
1930
1931 /* Offset into the user provided data */
1932 size_t offset;
1933
1934 /* Stride between elements into the user provided data */
1935 size_t stride;
1936 };
1937
1938 struct anv_descriptor_update_template {
1939 VkPipelineBindPoint bind_point;
1940
1941 /* The descriptor set this template corresponds to. This value is only
1942 * valid if the template was created with the templateType
1943 * VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET.
1944 */
1945 uint8_t set;
1946
1947 /* Number of entries in this template */
1948 uint32_t entry_count;
1949
1950 /* Entries of the template */
1951 struct anv_descriptor_template_entry entries[0];
1952 };
1953
1954 size_t
1955 anv_descriptor_set_layout_size(const struct anv_descriptor_set_layout *layout);
1956
1957 void
1958 anv_descriptor_set_write_image_view(struct anv_device *device,
1959 struct anv_descriptor_set *set,
1960 const VkDescriptorImageInfo * const info,
1961 VkDescriptorType type,
1962 uint32_t binding,
1963 uint32_t element);
1964
1965 void
1966 anv_descriptor_set_write_buffer_view(struct anv_device *device,
1967 struct anv_descriptor_set *set,
1968 VkDescriptorType type,
1969 struct anv_buffer_view *buffer_view,
1970 uint32_t binding,
1971 uint32_t element);
1972
1973 void
1974 anv_descriptor_set_write_buffer(struct anv_device *device,
1975 struct anv_descriptor_set *set,
1976 struct anv_state_stream *alloc_stream,
1977 VkDescriptorType type,
1978 struct anv_buffer *buffer,
1979 uint32_t binding,
1980 uint32_t element,
1981 VkDeviceSize offset,
1982 VkDeviceSize range);
1983 void
1984 anv_descriptor_set_write_inline_uniform_data(struct anv_device *device,
1985 struct anv_descriptor_set *set,
1986 uint32_t binding,
1987 const void *data,
1988 size_t offset,
1989 size_t size);
1990
1991 void
1992 anv_descriptor_set_write_template(struct anv_device *device,
1993 struct anv_descriptor_set *set,
1994 struct anv_state_stream *alloc_stream,
1995 const struct anv_descriptor_update_template *template,
1996 const void *data);
1997
1998 VkResult
1999 anv_descriptor_set_create(struct anv_device *device,
2000 struct anv_descriptor_pool *pool,
2001 struct anv_descriptor_set_layout *layout,
2002 struct anv_descriptor_set **out_set);
2003
2004 void
2005 anv_descriptor_set_destroy(struct anv_device *device,
2006 struct anv_descriptor_pool *pool,
2007 struct anv_descriptor_set *set);
2008
2009 #define ANV_DESCRIPTOR_SET_DESCRIPTORS (UINT8_MAX - 3)
2010 #define ANV_DESCRIPTOR_SET_NUM_WORK_GROUPS (UINT8_MAX - 2)
2011 #define ANV_DESCRIPTOR_SET_SHADER_CONSTANTS (UINT8_MAX - 1)
2012 #define ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS UINT8_MAX
2013
2014 struct anv_pipeline_binding {
2015 /* The descriptor set this surface corresponds to. The special value of
2016 * ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS indicates that the offset refers
2017 * to a color attachment and not a regular descriptor.
2018 */
2019 uint8_t set;
2020
2021 /* Binding in the descriptor set */
2022 uint32_t binding;
2023
2024 /* Index in the binding */
2025 uint32_t index;
2026
2027 /* Plane in the binding index */
2028 uint8_t plane;
2029
2030 /* Input attachment index (relative to the subpass) */
2031 uint8_t input_attachment_index;
2032
2033 /* For a storage image, whether it is write-only */
2034 bool write_only;
2035 };
2036
2037 struct anv_pipeline_layout {
2038 struct {
2039 struct anv_descriptor_set_layout *layout;
2040 uint32_t dynamic_offset_start;
2041 } set[MAX_SETS];
2042
2043 uint32_t num_sets;
2044
2045 unsigned char sha1[20];
2046 };
2047
2048 struct anv_buffer {
2049 struct anv_device * device;
2050 VkDeviceSize size;
2051
2052 VkBufferUsageFlags usage;
2053
2054 /* Set when bound */
2055 struct anv_address address;
2056 };
2057
2058 static inline uint64_t
2059 anv_buffer_get_range(struct anv_buffer *buffer, uint64_t offset, uint64_t range)
2060 {
2061 assert(offset <= buffer->size);
2062 if (range == VK_WHOLE_SIZE) {
2063 return buffer->size - offset;
2064 } else {
2065 assert(range + offset >= range);
2066 assert(range + offset <= buffer->size);
2067 return range;
2068 }
2069 }
2070
2071 enum anv_cmd_dirty_bits {
2072 ANV_CMD_DIRTY_DYNAMIC_VIEWPORT = 1 << 0, /* VK_DYNAMIC_STATE_VIEWPORT */
2073 ANV_CMD_DIRTY_DYNAMIC_SCISSOR = 1 << 1, /* VK_DYNAMIC_STATE_SCISSOR */
2074 ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH = 1 << 2, /* VK_DYNAMIC_STATE_LINE_WIDTH */
2075 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS = 1 << 3, /* VK_DYNAMIC_STATE_DEPTH_BIAS */
2076 ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS = 1 << 4, /* VK_DYNAMIC_STATE_BLEND_CONSTANTS */
2077 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS = 1 << 5, /* VK_DYNAMIC_STATE_DEPTH_BOUNDS */
2078 ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK = 1 << 6, /* VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK */
2079 ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK = 1 << 7, /* VK_DYNAMIC_STATE_STENCIL_WRITE_MASK */
2080 ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE = 1 << 8, /* VK_DYNAMIC_STATE_STENCIL_REFERENCE */
2081 ANV_CMD_DIRTY_PIPELINE = 1 << 9,
2082 ANV_CMD_DIRTY_INDEX_BUFFER = 1 << 10,
2083 ANV_CMD_DIRTY_RENDER_TARGETS = 1 << 11,
2084 ANV_CMD_DIRTY_XFB_ENABLE = 1 << 12,
2085 ANV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE = 1 << 13, /* VK_DYNAMIC_STATE_LINE_STIPPLE_EXT */
2086 };
2087 typedef uint32_t anv_cmd_dirty_mask_t;
2088
2089 #define ANV_CMD_DIRTY_DYNAMIC_ALL \
2090 (ANV_CMD_DIRTY_DYNAMIC_VIEWPORT | \
2091 ANV_CMD_DIRTY_DYNAMIC_SCISSOR | \
2092 ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH | \
2093 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS | \
2094 ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS | \
2095 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS | \
2096 ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK | \
2097 ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK | \
2098 ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE | \
2099 ANV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE)
2100
2101 static inline enum anv_cmd_dirty_bits
2102 anv_cmd_dirty_bit_for_vk_dynamic_state(VkDynamicState vk_state)
2103 {
2104 switch (vk_state) {
2105 case VK_DYNAMIC_STATE_VIEWPORT:
2106 return ANV_CMD_DIRTY_DYNAMIC_VIEWPORT;
2107 case VK_DYNAMIC_STATE_SCISSOR:
2108 return ANV_CMD_DIRTY_DYNAMIC_SCISSOR;
2109 case VK_DYNAMIC_STATE_LINE_WIDTH:
2110 return ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH;
2111 case VK_DYNAMIC_STATE_DEPTH_BIAS:
2112 return ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS;
2113 case VK_DYNAMIC_STATE_BLEND_CONSTANTS:
2114 return ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS;
2115 case VK_DYNAMIC_STATE_DEPTH_BOUNDS:
2116 return ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS;
2117 case VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK:
2118 return ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK;
2119 case VK_DYNAMIC_STATE_STENCIL_WRITE_MASK:
2120 return ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK;
2121 case VK_DYNAMIC_STATE_STENCIL_REFERENCE:
2122 return ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE;
2123 case VK_DYNAMIC_STATE_LINE_STIPPLE_EXT:
2124 return ANV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE;
2125 default:
2126 assert(!"Unsupported dynamic state");
2127 return 0;
2128 }
2129 }
2130
2131
2132 enum anv_pipe_bits {
2133 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT = (1 << 0),
2134 ANV_PIPE_STALL_AT_SCOREBOARD_BIT = (1 << 1),
2135 ANV_PIPE_STATE_CACHE_INVALIDATE_BIT = (1 << 2),
2136 ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT = (1 << 3),
2137 ANV_PIPE_VF_CACHE_INVALIDATE_BIT = (1 << 4),
2138 ANV_PIPE_DATA_CACHE_FLUSH_BIT = (1 << 5),
2139 ANV_PIPE_TILE_CACHE_FLUSH_BIT = (1 << 6),
2140 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT = (1 << 10),
2141 ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT = (1 << 11),
2142 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT = (1 << 12),
2143 ANV_PIPE_DEPTH_STALL_BIT = (1 << 13),
2144 ANV_PIPE_CS_STALL_BIT = (1 << 20),
2145
2146 /* This bit does not exist directly in PIPE_CONTROL. Instead it means that
2147 * a flush has happened but not a CS stall. The next time we do any sort
2148 * of invalidation we need to insert a CS stall at that time. Otherwise,
2149 * we would have to CS stall on every flush which could be bad.
2150 */
2151 ANV_PIPE_NEEDS_CS_STALL_BIT = (1 << 21),
2152
2153 /* This bit does not exist directly in PIPE_CONTROL. It means that render
2154 * target operations related to transfer commands with VkBuffer as
2155 * destination are ongoing. Some operations like copies on the command
2156 * streamer might need to be aware of this to trigger the appropriate stall
2157 * before they can proceed with the copy.
2158 */
2159 ANV_PIPE_RENDER_TARGET_BUFFER_WRITES = (1 << 22),
2160 };
2161
2162 #define ANV_PIPE_FLUSH_BITS ( \
2163 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | \
2164 ANV_PIPE_DATA_CACHE_FLUSH_BIT | \
2165 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | \
2166 ANV_PIPE_TILE_CACHE_FLUSH_BIT)
2167
2168 #define ANV_PIPE_STALL_BITS ( \
2169 ANV_PIPE_STALL_AT_SCOREBOARD_BIT | \
2170 ANV_PIPE_DEPTH_STALL_BIT | \
2171 ANV_PIPE_CS_STALL_BIT)
2172
2173 #define ANV_PIPE_INVALIDATE_BITS ( \
2174 ANV_PIPE_STATE_CACHE_INVALIDATE_BIT | \
2175 ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT | \
2176 ANV_PIPE_VF_CACHE_INVALIDATE_BIT | \
2177 ANV_PIPE_DATA_CACHE_FLUSH_BIT | \
2178 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT | \
2179 ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT)
2180
2181 static inline enum anv_pipe_bits
2182 anv_pipe_flush_bits_for_access_flags(VkAccessFlags flags)
2183 {
2184 enum anv_pipe_bits pipe_bits = 0;
2185
2186 unsigned b;
2187 for_each_bit(b, flags) {
2188 switch ((VkAccessFlagBits)(1 << b)) {
2189 case VK_ACCESS_SHADER_WRITE_BIT:
2190 /* We're transitioning a buffer that was previously used as write
2191 * destination through the data port. To make its content available
2192 * to future operations, flush the data cache.
2193 */
2194 pipe_bits |= ANV_PIPE_DATA_CACHE_FLUSH_BIT;
2195 break;
2196 case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT:
2197 /* We're transitioning a buffer that was previously used as render
2198 * target. To make its content available to future operations, flush
2199 * the render target cache.
2200 */
2201 pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
2202 break;
2203 case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT:
2204 /* We're transitioning a buffer that was previously used as depth
2205 * buffer. To make its content available to future operations, flush
2206 * the depth cache.
2207 */
2208 pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
2209 break;
2210 case VK_ACCESS_TRANSFER_WRITE_BIT:
2211 /* We're transitioning a buffer that was previously used as a
2212 * transfer write destination. Generic write operations include color
2213 * & depth operations as well as buffer operations like :
2214 * - vkCmdClearColorImage()
2215 * - vkCmdClearDepthStencilImage()
2216 * - vkCmdBlitImage()
2217 * - vkCmdCopy*(), vkCmdUpdate*(), vkCmdFill*()
2218 *
2219 * Most of these operations are implemented using Blorp which writes
2220 * through the render target, so flush that cache to make it visible
2221 * to future operations. And for depth related operations we also
2222 * need to flush the depth cache.
2223 */
2224 pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
2225 pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
2226 break;
2227 case VK_ACCESS_MEMORY_WRITE_BIT:
2228 /* We're transitioning a buffer for generic write operations. Flush
2229 * all the caches.
2230 */
2231 pipe_bits |= ANV_PIPE_FLUSH_BITS;
2232 break;
2233 default:
2234 break; /* Nothing to do */
2235 }
2236 }
2237
2238 return pipe_bits;
2239 }
2240
2241 static inline enum anv_pipe_bits
2242 anv_pipe_invalidate_bits_for_access_flags(VkAccessFlags flags)
2243 {
2244 enum anv_pipe_bits pipe_bits = 0;
2245
2246 unsigned b;
2247 for_each_bit(b, flags) {
2248 switch ((VkAccessFlagBits)(1 << b)) {
2249 case VK_ACCESS_INDIRECT_COMMAND_READ_BIT:
2250 /* Indirect draw commands take a buffer as input that we're going to
2251 * read from the command streamer to load some of the HW registers
2252 * (see genX_cmd_buffer.c:load_indirect_parameters). This requires a
2253 * command streamer stall so that all the cache flushes have
2254 * completed before the command streamer loads from memory.
2255 */
2256 pipe_bits |= ANV_PIPE_CS_STALL_BIT;
2257 /* Indirect draw commands also set gl_BaseVertex & gl_BaseIndex
2258 * through a vertex buffer, so invalidate that cache.
2259 */
2260 pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
2261 /* For CmdDipatchIndirect, we also load gl_NumWorkGroups through a
2262 * UBO from the buffer, so we need to invalidate constant cache.
2263 */
2264 pipe_bits |= ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
2265 break;
2266 case VK_ACCESS_INDEX_READ_BIT:
2267 case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT:
2268 /* We transitioning a buffer to be used for as input for vkCmdDraw*
2269 * commands, so we invalidate the VF cache to make sure there is no
2270 * stale data when we start rendering.
2271 */
2272 pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
2273 break;
2274 case VK_ACCESS_UNIFORM_READ_BIT:
2275 /* We transitioning a buffer to be used as uniform data. Because
2276 * uniform is accessed through the data port & sampler, we need to
2277 * invalidate the texture cache (sampler) & constant cache (data
2278 * port) to avoid stale data.
2279 */
2280 pipe_bits |= ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
2281 pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
2282 break;
2283 case VK_ACCESS_SHADER_READ_BIT:
2284 case VK_ACCESS_INPUT_ATTACHMENT_READ_BIT:
2285 case VK_ACCESS_TRANSFER_READ_BIT:
2286 /* Transitioning a buffer to be read through the sampler, so
2287 * invalidate the texture cache, we don't want any stale data.
2288 */
2289 pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
2290 break;
2291 case VK_ACCESS_MEMORY_READ_BIT:
2292 /* Transitioning a buffer for generic read, invalidate all the
2293 * caches.
2294 */
2295 pipe_bits |= ANV_PIPE_INVALIDATE_BITS;
2296 break;
2297 case VK_ACCESS_MEMORY_WRITE_BIT:
2298 /* Generic write, make sure all previously written things land in
2299 * memory.
2300 */
2301 pipe_bits |= ANV_PIPE_FLUSH_BITS;
2302 break;
2303 case VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT:
2304 /* Transitioning a buffer for conditional rendering. We'll load the
2305 * content of this buffer into HW registers using the command
2306 * streamer, so we need to stall the command streamer to make sure
2307 * any in-flight flush operations have completed.
2308 */
2309 pipe_bits |= ANV_PIPE_CS_STALL_BIT;
2310 break;
2311 default:
2312 break; /* Nothing to do */
2313 }
2314 }
2315
2316 return pipe_bits;
2317 }
2318
2319 #define VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV ( \
2320 VK_IMAGE_ASPECT_COLOR_BIT | \
2321 VK_IMAGE_ASPECT_PLANE_0_BIT | \
2322 VK_IMAGE_ASPECT_PLANE_1_BIT | \
2323 VK_IMAGE_ASPECT_PLANE_2_BIT)
2324 #define VK_IMAGE_ASPECT_PLANES_BITS_ANV ( \
2325 VK_IMAGE_ASPECT_PLANE_0_BIT | \
2326 VK_IMAGE_ASPECT_PLANE_1_BIT | \
2327 VK_IMAGE_ASPECT_PLANE_2_BIT)
2328
2329 struct anv_vertex_binding {
2330 struct anv_buffer * buffer;
2331 VkDeviceSize offset;
2332 };
2333
2334 struct anv_xfb_binding {
2335 struct anv_buffer * buffer;
2336 VkDeviceSize offset;
2337 VkDeviceSize size;
2338 };
2339
2340 #define ANV_PARAM_PUSH(offset) ((1 << 16) | (uint32_t)(offset))
2341 #define ANV_PARAM_IS_PUSH(param) ((uint32_t)(param) >> 16 == 1)
2342 #define ANV_PARAM_PUSH_OFFSET(param) ((param) & 0xffff)
2343
2344 #define ANV_PARAM_DYN_OFFSET(offset) ((2 << 16) | (uint32_t)(offset))
2345 #define ANV_PARAM_IS_DYN_OFFSET(param) ((uint32_t)(param) >> 16 == 2)
2346 #define ANV_PARAM_DYN_OFFSET_IDX(param) ((param) & 0xffff)
2347
2348 struct anv_push_constants {
2349 /* Push constant data provided by the client through vkPushConstants */
2350 uint8_t client_data[MAX_PUSH_CONSTANTS_SIZE];
2351
2352 /* Used for vkCmdDispatchBase */
2353 uint32_t base_work_group_id[3];
2354 };
2355
2356 struct anv_dynamic_state {
2357 struct {
2358 uint32_t count;
2359 VkViewport viewports[MAX_VIEWPORTS];
2360 } viewport;
2361
2362 struct {
2363 uint32_t count;
2364 VkRect2D scissors[MAX_SCISSORS];
2365 } scissor;
2366
2367 float line_width;
2368
2369 struct {
2370 float bias;
2371 float clamp;
2372 float slope;
2373 } depth_bias;
2374
2375 float blend_constants[4];
2376
2377 struct {
2378 float min;
2379 float max;
2380 } depth_bounds;
2381
2382 struct {
2383 uint32_t front;
2384 uint32_t back;
2385 } stencil_compare_mask;
2386
2387 struct {
2388 uint32_t front;
2389 uint32_t back;
2390 } stencil_write_mask;
2391
2392 struct {
2393 uint32_t front;
2394 uint32_t back;
2395 } stencil_reference;
2396
2397 struct {
2398 uint32_t factor;
2399 uint16_t pattern;
2400 } line_stipple;
2401 };
2402
2403 extern const struct anv_dynamic_state default_dynamic_state;
2404
2405 uint32_t anv_dynamic_state_copy(struct anv_dynamic_state *dest,
2406 const struct anv_dynamic_state *src,
2407 uint32_t copy_mask);
2408
2409 struct anv_surface_state {
2410 struct anv_state state;
2411 /** Address of the surface referred to by this state
2412 *
2413 * This address is relative to the start of the BO.
2414 */
2415 struct anv_address address;
2416 /* Address of the aux surface, if any
2417 *
2418 * This field is ANV_NULL_ADDRESS if and only if no aux surface exists.
2419 *
2420 * With the exception of gen8, the bottom 12 bits of this address' offset
2421 * include extra aux information.
2422 */
2423 struct anv_address aux_address;
2424 /* Address of the clear color, if any
2425 *
2426 * This address is relative to the start of the BO.
2427 */
2428 struct anv_address clear_address;
2429 };
2430
2431 /**
2432 * Attachment state when recording a renderpass instance.
2433 *
2434 * The clear value is valid only if there exists a pending clear.
2435 */
2436 struct anv_attachment_state {
2437 enum isl_aux_usage aux_usage;
2438 enum isl_aux_usage input_aux_usage;
2439 struct anv_surface_state color;
2440 struct anv_surface_state input;
2441
2442 VkImageLayout current_layout;
2443 VkImageLayout current_stencil_layout;
2444 VkImageAspectFlags pending_clear_aspects;
2445 VkImageAspectFlags pending_load_aspects;
2446 bool fast_clear;
2447 VkClearValue clear_value;
2448 bool clear_color_is_zero_one;
2449 bool clear_color_is_zero;
2450
2451 /* When multiview is active, attachments with a renderpass clear
2452 * operation have their respective layers cleared on the first
2453 * subpass that uses them, and only in that subpass. We keep track
2454 * of this using a bitfield to indicate which layers of an attachment
2455 * have not been cleared yet when multiview is active.
2456 */
2457 uint32_t pending_clear_views;
2458 struct anv_image_view * image_view;
2459 };
2460
2461 /** State tracking for particular pipeline bind point
2462 *
2463 * This struct is the base struct for anv_cmd_graphics_state and
2464 * anv_cmd_compute_state. These are used to track state which is bound to a
2465 * particular type of pipeline. Generic state that applies per-stage such as
2466 * binding table offsets and push constants is tracked generically with a
2467 * per-stage array in anv_cmd_state.
2468 */
2469 struct anv_cmd_pipeline_state {
2470 struct anv_pipeline *pipeline;
2471 struct anv_pipeline_layout *layout;
2472
2473 struct anv_descriptor_set *descriptors[MAX_SETS];
2474 uint32_t dynamic_offsets[MAX_DYNAMIC_BUFFERS];
2475
2476 struct anv_push_descriptor_set *push_descriptors[MAX_SETS];
2477 };
2478
2479 /** State tracking for graphics pipeline
2480 *
2481 * This has anv_cmd_pipeline_state as a base struct to track things which get
2482 * bound to a graphics pipeline. Along with general pipeline bind point state
2483 * which is in the anv_cmd_pipeline_state base struct, it also contains other
2484 * state which is graphics-specific.
2485 */
2486 struct anv_cmd_graphics_state {
2487 struct anv_cmd_pipeline_state base;
2488
2489 anv_cmd_dirty_mask_t dirty;
2490 uint32_t vb_dirty;
2491
2492 struct anv_dynamic_state dynamic;
2493
2494 struct {
2495 struct anv_buffer *index_buffer;
2496 uint32_t index_type; /**< 3DSTATE_INDEX_BUFFER.IndexFormat */
2497 uint32_t index_offset;
2498 } gen7;
2499 };
2500
2501 /** State tracking for compute pipeline
2502 *
2503 * This has anv_cmd_pipeline_state as a base struct to track things which get
2504 * bound to a compute pipeline. Along with general pipeline bind point state
2505 * which is in the anv_cmd_pipeline_state base struct, it also contains other
2506 * state which is compute-specific.
2507 */
2508 struct anv_cmd_compute_state {
2509 struct anv_cmd_pipeline_state base;
2510
2511 bool pipeline_dirty;
2512
2513 struct anv_address num_workgroups;
2514 };
2515
2516 /** State required while building cmd buffer */
2517 struct anv_cmd_state {
2518 /* PIPELINE_SELECT.PipelineSelection */
2519 uint32_t current_pipeline;
2520 const struct gen_l3_config * current_l3_config;
2521 uint32_t last_aux_map_state;
2522
2523 struct anv_cmd_graphics_state gfx;
2524 struct anv_cmd_compute_state compute;
2525
2526 enum anv_pipe_bits pending_pipe_bits;
2527 VkShaderStageFlags descriptors_dirty;
2528 VkShaderStageFlags push_constants_dirty;
2529
2530 struct anv_framebuffer * framebuffer;
2531 struct anv_render_pass * pass;
2532 struct anv_subpass * subpass;
2533 VkRect2D render_area;
2534 uint32_t restart_index;
2535 struct anv_vertex_binding vertex_bindings[MAX_VBS];
2536 bool xfb_enabled;
2537 struct anv_xfb_binding xfb_bindings[MAX_XFB_BUFFERS];
2538 VkShaderStageFlags push_constant_stages;
2539 struct anv_push_constants push_constants[MESA_SHADER_STAGES];
2540 struct anv_state binding_tables[MESA_SHADER_STAGES];
2541 struct anv_state samplers[MESA_SHADER_STAGES];
2542
2543 /**
2544 * Whether or not the gen8 PMA fix is enabled. We ensure that, at the top
2545 * of any command buffer it is disabled by disabling it in EndCommandBuffer
2546 * and before invoking the secondary in ExecuteCommands.
2547 */
2548 bool pma_fix_enabled;
2549
2550 /**
2551 * Whether or not we know for certain that HiZ is enabled for the current
2552 * subpass. If, for whatever reason, we are unsure as to whether HiZ is
2553 * enabled or not, this will be false.
2554 */
2555 bool hiz_enabled;
2556
2557 bool conditional_render_enabled;
2558
2559 /**
2560 * Last rendering scale argument provided to
2561 * genX(cmd_buffer_emit_hashing_mode)().
2562 */
2563 unsigned current_hash_scale;
2564
2565 /**
2566 * Array length is anv_cmd_state::pass::attachment_count. Array content is
2567 * valid only when recording a render pass instance.
2568 */
2569 struct anv_attachment_state * attachments;
2570
2571 /**
2572 * Surface states for color render targets. These are stored in a single
2573 * flat array. For depth-stencil attachments, the surface state is simply
2574 * left blank.
2575 */
2576 struct anv_state render_pass_states;
2577
2578 /**
2579 * A null surface state of the right size to match the framebuffer. This
2580 * is one of the states in render_pass_states.
2581 */
2582 struct anv_state null_surface_state;
2583 };
2584
2585 struct anv_cmd_pool {
2586 VkAllocationCallbacks alloc;
2587 struct list_head cmd_buffers;
2588 };
2589
2590 #define ANV_CMD_BUFFER_BATCH_SIZE 8192
2591
2592 enum anv_cmd_buffer_exec_mode {
2593 ANV_CMD_BUFFER_EXEC_MODE_PRIMARY,
2594 ANV_CMD_BUFFER_EXEC_MODE_EMIT,
2595 ANV_CMD_BUFFER_EXEC_MODE_GROW_AND_EMIT,
2596 ANV_CMD_BUFFER_EXEC_MODE_CHAIN,
2597 ANV_CMD_BUFFER_EXEC_MODE_COPY_AND_CHAIN,
2598 };
2599
2600 struct anv_cmd_buffer {
2601 VK_LOADER_DATA _loader_data;
2602
2603 struct anv_device * device;
2604
2605 struct anv_cmd_pool * pool;
2606 struct list_head pool_link;
2607
2608 struct anv_batch batch;
2609
2610 /* Fields required for the actual chain of anv_batch_bo's.
2611 *
2612 * These fields are initialized by anv_cmd_buffer_init_batch_bo_chain().
2613 */
2614 struct list_head batch_bos;
2615 enum anv_cmd_buffer_exec_mode exec_mode;
2616
2617 /* A vector of anv_batch_bo pointers for every batch or surface buffer
2618 * referenced by this command buffer
2619 *
2620 * initialized by anv_cmd_buffer_init_batch_bo_chain()
2621 */
2622 struct u_vector seen_bbos;
2623
2624 /* A vector of int32_t's for every block of binding tables.
2625 *
2626 * initialized by anv_cmd_buffer_init_batch_bo_chain()
2627 */
2628 struct u_vector bt_block_states;
2629 uint32_t bt_next;
2630
2631 struct anv_reloc_list surface_relocs;
2632 /** Last seen surface state block pool center bo offset */
2633 uint32_t last_ss_pool_center;
2634
2635 /* Serial for tracking buffer completion */
2636 uint32_t serial;
2637
2638 /* Stream objects for storing temporary data */
2639 struct anv_state_stream surface_state_stream;
2640 struct anv_state_stream dynamic_state_stream;
2641
2642 VkCommandBufferUsageFlags usage_flags;
2643 VkCommandBufferLevel level;
2644
2645 struct anv_cmd_state state;
2646
2647 /* Set by SetPerformanceMarkerINTEL, written into queries by CmdBeginQuery */
2648 uint64_t intel_perf_marker;
2649 };
2650
2651 VkResult anv_cmd_buffer_init_batch_bo_chain(struct anv_cmd_buffer *cmd_buffer);
2652 void anv_cmd_buffer_fini_batch_bo_chain(struct anv_cmd_buffer *cmd_buffer);
2653 void anv_cmd_buffer_reset_batch_bo_chain(struct anv_cmd_buffer *cmd_buffer);
2654 void anv_cmd_buffer_end_batch_buffer(struct anv_cmd_buffer *cmd_buffer);
2655 void anv_cmd_buffer_add_secondary(struct anv_cmd_buffer *primary,
2656 struct anv_cmd_buffer *secondary);
2657 void anv_cmd_buffer_prepare_execbuf(struct anv_cmd_buffer *cmd_buffer);
2658 VkResult anv_cmd_buffer_execbuf(struct anv_queue *queue,
2659 struct anv_cmd_buffer *cmd_buffer,
2660 const VkSemaphore *in_semaphores,
2661 const uint64_t *in_wait_values,
2662 uint32_t num_in_semaphores,
2663 const VkSemaphore *out_semaphores,
2664 const uint64_t *out_signal_values,
2665 uint32_t num_out_semaphores,
2666 VkFence fence);
2667
2668 VkResult anv_cmd_buffer_reset(struct anv_cmd_buffer *cmd_buffer);
2669
2670 struct anv_state anv_cmd_buffer_emit_dynamic(struct anv_cmd_buffer *cmd_buffer,
2671 const void *data, uint32_t size, uint32_t alignment);
2672 struct anv_state anv_cmd_buffer_merge_dynamic(struct anv_cmd_buffer *cmd_buffer,
2673 uint32_t *a, uint32_t *b,
2674 uint32_t dwords, uint32_t alignment);
2675
2676 struct anv_address
2677 anv_cmd_buffer_surface_base_address(struct anv_cmd_buffer *cmd_buffer);
2678 struct anv_state
2679 anv_cmd_buffer_alloc_binding_table(struct anv_cmd_buffer *cmd_buffer,
2680 uint32_t entries, uint32_t *state_offset);
2681 struct anv_state
2682 anv_cmd_buffer_alloc_surface_state(struct anv_cmd_buffer *cmd_buffer);
2683 struct anv_state
2684 anv_cmd_buffer_alloc_dynamic_state(struct anv_cmd_buffer *cmd_buffer,
2685 uint32_t size, uint32_t alignment);
2686
2687 VkResult
2688 anv_cmd_buffer_new_binding_table_block(struct anv_cmd_buffer *cmd_buffer);
2689
2690 void gen8_cmd_buffer_emit_viewport(struct anv_cmd_buffer *cmd_buffer);
2691 void gen8_cmd_buffer_emit_depth_viewport(struct anv_cmd_buffer *cmd_buffer,
2692 bool depth_clamp_enable);
2693 void gen7_cmd_buffer_emit_scissor(struct anv_cmd_buffer *cmd_buffer);
2694
2695 void anv_cmd_buffer_setup_attachments(struct anv_cmd_buffer *cmd_buffer,
2696 struct anv_render_pass *pass,
2697 struct anv_framebuffer *framebuffer,
2698 const VkClearValue *clear_values);
2699
2700 void anv_cmd_buffer_emit_state_base_address(struct anv_cmd_buffer *cmd_buffer);
2701
2702 struct anv_state
2703 anv_cmd_buffer_push_constants(struct anv_cmd_buffer *cmd_buffer,
2704 gl_shader_stage stage);
2705 struct anv_state
2706 anv_cmd_buffer_cs_push_constants(struct anv_cmd_buffer *cmd_buffer);
2707
2708 const struct anv_image_view *
2709 anv_cmd_buffer_get_depth_stencil_view(const struct anv_cmd_buffer *cmd_buffer);
2710
2711 VkResult
2712 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
2713 uint32_t num_entries,
2714 uint32_t *state_offset,
2715 struct anv_state *bt_state);
2716
2717 void anv_cmd_buffer_dump(struct anv_cmd_buffer *cmd_buffer);
2718
2719 void anv_cmd_emit_conditional_render_predicate(struct anv_cmd_buffer *cmd_buffer);
2720
2721 enum anv_fence_type {
2722 ANV_FENCE_TYPE_NONE = 0,
2723 ANV_FENCE_TYPE_BO,
2724 ANV_FENCE_TYPE_SYNCOBJ,
2725 ANV_FENCE_TYPE_WSI,
2726 };
2727
2728 enum anv_bo_fence_state {
2729 /** Indicates that this is a new (or newly reset fence) */
2730 ANV_BO_FENCE_STATE_RESET,
2731
2732 /** Indicates that this fence has been submitted to the GPU but is still
2733 * (as far as we know) in use by the GPU.
2734 */
2735 ANV_BO_FENCE_STATE_SUBMITTED,
2736
2737 ANV_BO_FENCE_STATE_SIGNALED,
2738 };
2739
2740 struct anv_fence_impl {
2741 enum anv_fence_type type;
2742
2743 union {
2744 /** Fence implementation for BO fences
2745 *
2746 * These fences use a BO and a set of CPU-tracked state flags. The BO
2747 * is added to the object list of the last execbuf call in a QueueSubmit
2748 * and is marked EXEC_WRITE. The state flags track when the BO has been
2749 * submitted to the kernel. We need to do this because Vulkan lets you
2750 * wait on a fence that has not yet been submitted and I915_GEM_BUSY
2751 * will say it's idle in this case.
2752 */
2753 struct {
2754 struct anv_bo *bo;
2755 enum anv_bo_fence_state state;
2756 } bo;
2757
2758 /** DRM syncobj handle for syncobj-based fences */
2759 uint32_t syncobj;
2760
2761 /** WSI fence */
2762 struct wsi_fence *fence_wsi;
2763 };
2764 };
2765
2766 struct anv_fence {
2767 /* Permanent fence state. Every fence has some form of permanent state
2768 * (type != ANV_SEMAPHORE_TYPE_NONE). This may be a BO to fence on (for
2769 * cross-process fences) or it could just be a dummy for use internally.
2770 */
2771 struct anv_fence_impl permanent;
2772
2773 /* Temporary fence state. A fence *may* have temporary state. That state
2774 * is added to the fence by an import operation and is reset back to
2775 * ANV_SEMAPHORE_TYPE_NONE when the fence is reset. A fence with temporary
2776 * state cannot be signaled because the fence must already be signaled
2777 * before the temporary state can be exported from the fence in the other
2778 * process and imported here.
2779 */
2780 struct anv_fence_impl temporary;
2781 };
2782
2783 struct anv_event {
2784 uint64_t semaphore;
2785 struct anv_state state;
2786 };
2787
2788 enum anv_semaphore_type {
2789 ANV_SEMAPHORE_TYPE_NONE = 0,
2790 ANV_SEMAPHORE_TYPE_DUMMY,
2791 ANV_SEMAPHORE_TYPE_BO,
2792 ANV_SEMAPHORE_TYPE_SYNC_FILE,
2793 ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ,
2794 ANV_SEMAPHORE_TYPE_TIMELINE,
2795 };
2796
2797 struct anv_timeline_point {
2798 struct list_head link;
2799
2800 uint64_t serial;
2801
2802 /* Number of waiter on this point, when > 0 the point should not be garbage
2803 * collected.
2804 */
2805 int waiting;
2806
2807 /* BO used for synchronization. */
2808 struct anv_bo *bo;
2809 };
2810
2811 struct anv_timeline {
2812 pthread_mutex_t mutex;
2813 pthread_cond_t cond;
2814
2815 uint64_t highest_past;
2816 uint64_t highest_pending;
2817
2818 struct list_head points;
2819 struct list_head free_points;
2820 };
2821
2822 struct anv_semaphore_impl {
2823 enum anv_semaphore_type type;
2824
2825 union {
2826 /* A BO representing this semaphore when type == ANV_SEMAPHORE_TYPE_BO.
2827 * This BO will be added to the object list on any execbuf2 calls for
2828 * which this semaphore is used as a wait or signal fence. When used as
2829 * a signal fence, the EXEC_OBJECT_WRITE flag will be set.
2830 */
2831 struct anv_bo *bo;
2832
2833 /* The sync file descriptor when type == ANV_SEMAPHORE_TYPE_SYNC_FILE.
2834 * If the semaphore is in the unsignaled state due to either just being
2835 * created or because it has been used for a wait, fd will be -1.
2836 */
2837 int fd;
2838
2839 /* Sync object handle when type == ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ.
2840 * Unlike GEM BOs, DRM sync objects aren't deduplicated by the kernel on
2841 * import so we don't need to bother with a userspace cache.
2842 */
2843 uint32_t syncobj;
2844
2845 /* Non shareable timeline semaphore
2846 *
2847 * Used when kernel don't have support for timeline semaphores.
2848 */
2849 struct anv_timeline timeline;
2850 };
2851 };
2852
2853 struct anv_semaphore {
2854 uint32_t refcount;
2855
2856 /* Permanent semaphore state. Every semaphore has some form of permanent
2857 * state (type != ANV_SEMAPHORE_TYPE_NONE). This may be a BO to fence on
2858 * (for cross-process semaphores0 or it could just be a dummy for use
2859 * internally.
2860 */
2861 struct anv_semaphore_impl permanent;
2862
2863 /* Temporary semaphore state. A semaphore *may* have temporary state.
2864 * That state is added to the semaphore by an import operation and is reset
2865 * back to ANV_SEMAPHORE_TYPE_NONE when the semaphore is waited on. A
2866 * semaphore with temporary state cannot be signaled because the semaphore
2867 * must already be signaled before the temporary state can be exported from
2868 * the semaphore in the other process and imported here.
2869 */
2870 struct anv_semaphore_impl temporary;
2871 };
2872
2873 void anv_semaphore_reset_temporary(struct anv_device *device,
2874 struct anv_semaphore *semaphore);
2875
2876 struct anv_shader_module {
2877 unsigned char sha1[20];
2878 uint32_t size;
2879 char data[0];
2880 };
2881
2882 static inline gl_shader_stage
2883 vk_to_mesa_shader_stage(VkShaderStageFlagBits vk_stage)
2884 {
2885 assert(__builtin_popcount(vk_stage) == 1);
2886 return ffs(vk_stage) - 1;
2887 }
2888
2889 static inline VkShaderStageFlagBits
2890 mesa_to_vk_shader_stage(gl_shader_stage mesa_stage)
2891 {
2892 return (1 << mesa_stage);
2893 }
2894
2895 #define ANV_STAGE_MASK ((1 << MESA_SHADER_STAGES) - 1)
2896
2897 #define anv_foreach_stage(stage, stage_bits) \
2898 for (gl_shader_stage stage, \
2899 __tmp = (gl_shader_stage)((stage_bits) & ANV_STAGE_MASK); \
2900 stage = __builtin_ffs(__tmp) - 1, __tmp; \
2901 __tmp &= ~(1 << (stage)))
2902
2903 struct anv_pipeline_bind_map {
2904 uint32_t surface_count;
2905 uint32_t sampler_count;
2906
2907 struct anv_pipeline_binding * surface_to_descriptor;
2908 struct anv_pipeline_binding * sampler_to_descriptor;
2909 };
2910
2911 struct anv_shader_bin_key {
2912 uint32_t size;
2913 uint8_t data[0];
2914 };
2915
2916 struct anv_shader_bin {
2917 uint32_t ref_cnt;
2918
2919 const struct anv_shader_bin_key *key;
2920
2921 struct anv_state kernel;
2922 uint32_t kernel_size;
2923
2924 struct anv_state constant_data;
2925 uint32_t constant_data_size;
2926
2927 const struct brw_stage_prog_data *prog_data;
2928 uint32_t prog_data_size;
2929
2930 struct brw_compile_stats stats[3];
2931 uint32_t num_stats;
2932
2933 struct nir_xfb_info *xfb_info;
2934
2935 struct anv_pipeline_bind_map bind_map;
2936 };
2937
2938 struct anv_shader_bin *
2939 anv_shader_bin_create(struct anv_device *device,
2940 const void *key, uint32_t key_size,
2941 const void *kernel, uint32_t kernel_size,
2942 const void *constant_data, uint32_t constant_data_size,
2943 const struct brw_stage_prog_data *prog_data,
2944 uint32_t prog_data_size, const void *prog_data_param,
2945 const struct brw_compile_stats *stats, uint32_t num_stats,
2946 const struct nir_xfb_info *xfb_info,
2947 const struct anv_pipeline_bind_map *bind_map);
2948
2949 void
2950 anv_shader_bin_destroy(struct anv_device *device, struct anv_shader_bin *shader);
2951
2952 static inline void
2953 anv_shader_bin_ref(struct anv_shader_bin *shader)
2954 {
2955 assert(shader && shader->ref_cnt >= 1);
2956 p_atomic_inc(&shader->ref_cnt);
2957 }
2958
2959 static inline void
2960 anv_shader_bin_unref(struct anv_device *device, struct anv_shader_bin *shader)
2961 {
2962 assert(shader && shader->ref_cnt >= 1);
2963 if (p_atomic_dec_zero(&shader->ref_cnt))
2964 anv_shader_bin_destroy(device, shader);
2965 }
2966
2967 /* 5 possible simultaneous shader stages and FS may have up to 3 binaries */
2968 #define MAX_PIPELINE_EXECUTABLES 7
2969
2970 struct anv_pipeline_executable {
2971 gl_shader_stage stage;
2972
2973 struct brw_compile_stats stats;
2974
2975 char *nir;
2976 char *disasm;
2977 };
2978
2979 struct anv_pipeline {
2980 struct anv_device * device;
2981 struct anv_batch batch;
2982 uint32_t batch_data[512];
2983 struct anv_reloc_list batch_relocs;
2984 anv_cmd_dirty_mask_t dynamic_state_mask;
2985 struct anv_dynamic_state dynamic_state;
2986
2987 void * mem_ctx;
2988
2989 VkPipelineCreateFlags flags;
2990 struct anv_subpass * subpass;
2991
2992 bool needs_data_cache;
2993
2994 struct anv_shader_bin * shaders[MESA_SHADER_STAGES];
2995
2996 uint32_t num_executables;
2997 struct anv_pipeline_executable executables[MAX_PIPELINE_EXECUTABLES];
2998
2999 struct {
3000 const struct gen_l3_config * l3_config;
3001 uint32_t total_size;
3002 } urb;
3003
3004 VkShaderStageFlags active_stages;
3005 struct anv_state blend_state;
3006
3007 uint32_t vb_used;
3008 struct anv_pipeline_vertex_binding {
3009 uint32_t stride;
3010 bool instanced;
3011 uint32_t instance_divisor;
3012 } vb[MAX_VBS];
3013
3014 uint8_t xfb_used;
3015
3016 bool primitive_restart;
3017 uint32_t topology;
3018
3019 uint32_t cs_right_mask;
3020
3021 bool writes_depth;
3022 bool depth_test_enable;
3023 bool writes_stencil;
3024 bool stencil_test_enable;
3025 bool depth_clamp_enable;
3026 bool depth_clip_enable;
3027 bool sample_shading_enable;
3028 bool kill_pixel;
3029 bool depth_bounds_test_enable;
3030
3031 struct {
3032 uint32_t sf[7];
3033 uint32_t depth_stencil_state[3];
3034 } gen7;
3035
3036 struct {
3037 uint32_t sf[4];
3038 uint32_t raster[5];
3039 uint32_t wm_depth_stencil[3];
3040 } gen8;
3041
3042 struct {
3043 uint32_t wm_depth_stencil[4];
3044 } gen9;
3045
3046 uint32_t interface_descriptor_data[8];
3047 };
3048
3049 static inline bool
3050 anv_pipeline_has_stage(const struct anv_pipeline *pipeline,
3051 gl_shader_stage stage)
3052 {
3053 return (pipeline->active_stages & mesa_to_vk_shader_stage(stage)) != 0;
3054 }
3055
3056 #define ANV_DECL_GET_PROG_DATA_FUNC(prefix, stage) \
3057 static inline const struct brw_##prefix##_prog_data * \
3058 get_##prefix##_prog_data(const struct anv_pipeline *pipeline) \
3059 { \
3060 if (anv_pipeline_has_stage(pipeline, stage)) { \
3061 return (const struct brw_##prefix##_prog_data *) \
3062 pipeline->shaders[stage]->prog_data; \
3063 } else { \
3064 return NULL; \
3065 } \
3066 }
3067
3068 ANV_DECL_GET_PROG_DATA_FUNC(vs, MESA_SHADER_VERTEX)
3069 ANV_DECL_GET_PROG_DATA_FUNC(tcs, MESA_SHADER_TESS_CTRL)
3070 ANV_DECL_GET_PROG_DATA_FUNC(tes, MESA_SHADER_TESS_EVAL)
3071 ANV_DECL_GET_PROG_DATA_FUNC(gs, MESA_SHADER_GEOMETRY)
3072 ANV_DECL_GET_PROG_DATA_FUNC(wm, MESA_SHADER_FRAGMENT)
3073 ANV_DECL_GET_PROG_DATA_FUNC(cs, MESA_SHADER_COMPUTE)
3074
3075 static inline const struct brw_vue_prog_data *
3076 anv_pipeline_get_last_vue_prog_data(const struct anv_pipeline *pipeline)
3077 {
3078 if (anv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY))
3079 return &get_gs_prog_data(pipeline)->base;
3080 else if (anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL))
3081 return &get_tes_prog_data(pipeline)->base;
3082 else
3083 return &get_vs_prog_data(pipeline)->base;
3084 }
3085
3086 VkResult
3087 anv_pipeline_init(struct anv_pipeline *pipeline, struct anv_device *device,
3088 struct anv_pipeline_cache *cache,
3089 const VkGraphicsPipelineCreateInfo *pCreateInfo,
3090 const VkAllocationCallbacks *alloc);
3091
3092 VkResult
3093 anv_pipeline_compile_cs(struct anv_pipeline *pipeline,
3094 struct anv_pipeline_cache *cache,
3095 const VkComputePipelineCreateInfo *info,
3096 const struct anv_shader_module *module,
3097 const char *entrypoint,
3098 const VkSpecializationInfo *spec_info);
3099
3100 struct anv_format_plane {
3101 enum isl_format isl_format:16;
3102 struct isl_swizzle swizzle;
3103
3104 /* Whether this plane contains chroma channels */
3105 bool has_chroma;
3106
3107 /* For downscaling of YUV planes */
3108 uint8_t denominator_scales[2];
3109
3110 /* How to map sampled ycbcr planes to a single 4 component element. */
3111 struct isl_swizzle ycbcr_swizzle;
3112
3113 /* What aspect is associated to this plane */
3114 VkImageAspectFlags aspect;
3115 };
3116
3117
3118 struct anv_format {
3119 struct anv_format_plane planes[3];
3120 VkFormat vk_format;
3121 uint8_t n_planes;
3122 bool can_ycbcr;
3123 };
3124
3125 static inline uint32_t
3126 anv_image_aspect_to_plane(VkImageAspectFlags image_aspects,
3127 VkImageAspectFlags aspect_mask)
3128 {
3129 switch (aspect_mask) {
3130 case VK_IMAGE_ASPECT_COLOR_BIT:
3131 case VK_IMAGE_ASPECT_DEPTH_BIT:
3132 case VK_IMAGE_ASPECT_PLANE_0_BIT:
3133 return 0;
3134 case VK_IMAGE_ASPECT_STENCIL_BIT:
3135 if ((image_aspects & VK_IMAGE_ASPECT_DEPTH_BIT) == 0)
3136 return 0;
3137 /* Fall-through */
3138 case VK_IMAGE_ASPECT_PLANE_1_BIT:
3139 return 1;
3140 case VK_IMAGE_ASPECT_PLANE_2_BIT:
3141 return 2;
3142 default:
3143 /* Purposefully assert with depth/stencil aspects. */
3144 unreachable("invalid image aspect");
3145 }
3146 }
3147
3148 static inline VkImageAspectFlags
3149 anv_plane_to_aspect(VkImageAspectFlags image_aspects,
3150 uint32_t plane)
3151 {
3152 if (image_aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
3153 if (util_bitcount(image_aspects) > 1)
3154 return VK_IMAGE_ASPECT_PLANE_0_BIT << plane;
3155 return VK_IMAGE_ASPECT_COLOR_BIT;
3156 }
3157 if (image_aspects & VK_IMAGE_ASPECT_DEPTH_BIT)
3158 return VK_IMAGE_ASPECT_DEPTH_BIT << plane;
3159 assert(image_aspects == VK_IMAGE_ASPECT_STENCIL_BIT);
3160 return VK_IMAGE_ASPECT_STENCIL_BIT;
3161 }
3162
3163 #define anv_foreach_image_aspect_bit(b, image, aspects) \
3164 for_each_bit(b, anv_image_expand_aspects(image, aspects))
3165
3166 const struct anv_format *
3167 anv_get_format(VkFormat format);
3168
3169 static inline uint32_t
3170 anv_get_format_planes(VkFormat vk_format)
3171 {
3172 const struct anv_format *format = anv_get_format(vk_format);
3173
3174 return format != NULL ? format->n_planes : 0;
3175 }
3176
3177 struct anv_format_plane
3178 anv_get_format_plane(const struct gen_device_info *devinfo, VkFormat vk_format,
3179 VkImageAspectFlagBits aspect, VkImageTiling tiling);
3180
3181 static inline enum isl_format
3182 anv_get_isl_format(const struct gen_device_info *devinfo, VkFormat vk_format,
3183 VkImageAspectFlags aspect, VkImageTiling tiling)
3184 {
3185 return anv_get_format_plane(devinfo, vk_format, aspect, tiling).isl_format;
3186 }
3187
3188 static inline struct isl_swizzle
3189 anv_swizzle_for_render(struct isl_swizzle swizzle)
3190 {
3191 /* Sometimes the swizzle will have alpha map to one. We do this to fake
3192 * RGB as RGBA for texturing
3193 */
3194 assert(swizzle.a == ISL_CHANNEL_SELECT_ONE ||
3195 swizzle.a == ISL_CHANNEL_SELECT_ALPHA);
3196
3197 /* But it doesn't matter what we render to that channel */
3198 swizzle.a = ISL_CHANNEL_SELECT_ALPHA;
3199
3200 return swizzle;
3201 }
3202
3203 void
3204 anv_pipeline_setup_l3_config(struct anv_pipeline *pipeline, bool needs_slm);
3205
3206 /**
3207 * Subsurface of an anv_image.
3208 */
3209 struct anv_surface {
3210 /** Valid only if isl_surf::size_B > 0. */
3211 struct isl_surf isl;
3212
3213 /**
3214 * Offset from VkImage's base address, as bound by vkBindImageMemory().
3215 */
3216 uint32_t offset;
3217 };
3218
3219 struct anv_image {
3220 VkImageType type; /**< VkImageCreateInfo::imageType */
3221 /* The original VkFormat provided by the client. This may not match any
3222 * of the actual surface formats.
3223 */
3224 VkFormat vk_format;
3225 const struct anv_format *format;
3226
3227 VkImageAspectFlags aspects;
3228 VkExtent3D extent;
3229 uint32_t levels;
3230 uint32_t array_size;
3231 uint32_t samples; /**< VkImageCreateInfo::samples */
3232 uint32_t n_planes;
3233 VkImageUsageFlags usage; /**< VkImageCreateInfo::usage. */
3234 VkImageUsageFlags stencil_usage;
3235 VkImageCreateFlags create_flags; /* Flags used when creating image. */
3236 VkImageTiling tiling; /** VkImageCreateInfo::tiling */
3237
3238 /** True if this is needs to be bound to an appropriately tiled BO.
3239 *
3240 * When not using modifiers, consumers such as X11, Wayland, and KMS need
3241 * the tiling passed via I915_GEM_SET_TILING. When exporting these buffers
3242 * we require a dedicated allocation so that we can know to allocate a
3243 * tiled buffer.
3244 */
3245 bool needs_set_tiling;
3246
3247 /**
3248 * Must be DRM_FORMAT_MOD_INVALID unless tiling is
3249 * VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT.
3250 */
3251 uint64_t drm_format_mod;
3252
3253 VkDeviceSize size;
3254 uint32_t alignment;
3255
3256 /* Whether the image is made of several underlying buffer objects rather a
3257 * single one with different offsets.
3258 */
3259 bool disjoint;
3260
3261 /* All the formats that can be used when creating views of this image
3262 * are CCS_E compatible.
3263 */
3264 bool ccs_e_compatible;
3265
3266 /* Image was created with external format. */
3267 bool external_format;
3268
3269 /**
3270 * Image subsurfaces
3271 *
3272 * For each foo, anv_image::planes[x].surface is valid if and only if
3273 * anv_image::aspects has a x aspect. Refer to anv_image_aspect_to_plane()
3274 * to figure the number associated with a given aspect.
3275 *
3276 * The hardware requires that the depth buffer and stencil buffer be
3277 * separate surfaces. From Vulkan's perspective, though, depth and stencil
3278 * reside in the same VkImage. To satisfy both the hardware and Vulkan, we
3279 * allocate the depth and stencil buffers as separate surfaces in the same
3280 * bo.
3281 *
3282 * Memory layout :
3283 *
3284 * -----------------------
3285 * | surface0 | /|\
3286 * ----------------------- |
3287 * | shadow surface0 | |
3288 * ----------------------- | Plane 0
3289 * | aux surface0 | |
3290 * ----------------------- |
3291 * | fast clear colors0 | \|/
3292 * -----------------------
3293 * | surface1 | /|\
3294 * ----------------------- |
3295 * | shadow surface1 | |
3296 * ----------------------- | Plane 1
3297 * | aux surface1 | |
3298 * ----------------------- |
3299 * | fast clear colors1 | \|/
3300 * -----------------------
3301 * | ... |
3302 * | |
3303 * -----------------------
3304 */
3305 struct {
3306 /**
3307 * Offset of the entire plane (whenever the image is disjoint this is
3308 * set to 0).
3309 */
3310 uint32_t offset;
3311
3312 VkDeviceSize size;
3313 uint32_t alignment;
3314
3315 struct anv_surface surface;
3316
3317 /**
3318 * A surface which shadows the main surface and may have different
3319 * tiling. This is used for sampling using a tiling that isn't supported
3320 * for other operations.
3321 */
3322 struct anv_surface shadow_surface;
3323
3324 /**
3325 * For color images, this is the aux usage for this image when not used
3326 * as a color attachment.
3327 *
3328 * For depth/stencil images, this is set to ISL_AUX_USAGE_HIZ if the
3329 * image has a HiZ buffer.
3330 */
3331 enum isl_aux_usage aux_usage;
3332
3333 struct anv_surface aux_surface;
3334
3335 /**
3336 * Offset of the fast clear state (used to compute the
3337 * fast_clear_state_offset of the following planes).
3338 */
3339 uint32_t fast_clear_state_offset;
3340
3341 /**
3342 * BO associated with this plane, set when bound.
3343 */
3344 struct anv_address address;
3345
3346 /**
3347 * Address of the main surface used to fill the aux map table. This is
3348 * used at destruction of the image since the Vulkan spec does not
3349 * guarantee that the address.bo field we still be valid at destruction.
3350 */
3351 uint64_t aux_map_surface_address;
3352
3353 /**
3354 * When destroying the image, also free the bo.
3355 * */
3356 bool bo_is_owned;
3357 } planes[3];
3358 };
3359
3360 /* The ordering of this enum is important */
3361 enum anv_fast_clear_type {
3362 /** Image does not have/support any fast-clear blocks */
3363 ANV_FAST_CLEAR_NONE = 0,
3364 /** Image has/supports fast-clear but only to the default value */
3365 ANV_FAST_CLEAR_DEFAULT_VALUE = 1,
3366 /** Image has/supports fast-clear with an arbitrary fast-clear value */
3367 ANV_FAST_CLEAR_ANY = 2,
3368 };
3369
3370 /* Returns the number of auxiliary buffer levels attached to an image. */
3371 static inline uint8_t
3372 anv_image_aux_levels(const struct anv_image * const image,
3373 VkImageAspectFlagBits aspect)
3374 {
3375 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
3376
3377 /* The Gen12 CCS aux surface is represented with only one level. */
3378 const uint8_t aux_logical_levels =
3379 image->planes[plane].aux_surface.isl.tiling == ISL_TILING_GEN12_CCS ?
3380 image->planes[plane].surface.isl.levels :
3381 image->planes[plane].aux_surface.isl.levels;
3382
3383 return image->planes[plane].aux_surface.isl.size_B > 0 ?
3384 aux_logical_levels : 0;
3385 }
3386
3387 /* Returns the number of auxiliary buffer layers attached to an image. */
3388 static inline uint32_t
3389 anv_image_aux_layers(const struct anv_image * const image,
3390 VkImageAspectFlagBits aspect,
3391 const uint8_t miplevel)
3392 {
3393 assert(image);
3394
3395 /* The miplevel must exist in the main buffer. */
3396 assert(miplevel < image->levels);
3397
3398 if (miplevel >= anv_image_aux_levels(image, aspect)) {
3399 /* There are no layers with auxiliary data because the miplevel has no
3400 * auxiliary data.
3401 */
3402 return 0;
3403 } else {
3404 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
3405
3406 /* The Gen12 CCS aux surface is represented with only one layer. */
3407 const struct isl_extent4d *aux_logical_level0_px =
3408 image->planes[plane].aux_surface.isl.tiling == ISL_TILING_GEN12_CCS ?
3409 &image->planes[plane].surface.isl.logical_level0_px :
3410 &image->planes[plane].aux_surface.isl.logical_level0_px;
3411
3412 return MAX2(aux_logical_level0_px->array_len,
3413 aux_logical_level0_px->depth >> miplevel);
3414 }
3415 }
3416
3417 static inline struct anv_address
3418 anv_image_get_clear_color_addr(const struct anv_device *device,
3419 const struct anv_image *image,
3420 VkImageAspectFlagBits aspect)
3421 {
3422 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
3423
3424 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
3425 return anv_address_add(image->planes[plane].address,
3426 image->planes[plane].fast_clear_state_offset);
3427 }
3428
3429 static inline struct anv_address
3430 anv_image_get_fast_clear_type_addr(const struct anv_device *device,
3431 const struct anv_image *image,
3432 VkImageAspectFlagBits aspect)
3433 {
3434 struct anv_address addr =
3435 anv_image_get_clear_color_addr(device, image, aspect);
3436
3437 const unsigned clear_color_state_size = device->info.gen >= 10 ?
3438 device->isl_dev.ss.clear_color_state_size :
3439 device->isl_dev.ss.clear_value_size;
3440 return anv_address_add(addr, clear_color_state_size);
3441 }
3442
3443 static inline struct anv_address
3444 anv_image_get_compression_state_addr(const struct anv_device *device,
3445 const struct anv_image *image,
3446 VkImageAspectFlagBits aspect,
3447 uint32_t level, uint32_t array_layer)
3448 {
3449 assert(level < anv_image_aux_levels(image, aspect));
3450 assert(array_layer < anv_image_aux_layers(image, aspect, level));
3451 UNUSED uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
3452 assert(image->planes[plane].aux_usage == ISL_AUX_USAGE_CCS_E);
3453
3454 struct anv_address addr =
3455 anv_image_get_fast_clear_type_addr(device, image, aspect);
3456 addr.offset += 4; /* Go past the fast clear type */
3457
3458 if (image->type == VK_IMAGE_TYPE_3D) {
3459 for (uint32_t l = 0; l < level; l++)
3460 addr.offset += anv_minify(image->extent.depth, l) * 4;
3461 } else {
3462 addr.offset += level * image->array_size * 4;
3463 }
3464 addr.offset += array_layer * 4;
3465
3466 assert(addr.offset <
3467 image->planes[plane].address.offset + image->planes[plane].size);
3468 return addr;
3469 }
3470
3471 /* Returns true if a HiZ-enabled depth buffer can be sampled from. */
3472 static inline bool
3473 anv_can_sample_with_hiz(const struct gen_device_info * const devinfo,
3474 const struct anv_image *image)
3475 {
3476 if (!(image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT))
3477 return false;
3478
3479 /* Allow this feature on BDW even though it is disabled in the BDW devinfo
3480 * struct. There's documentation which suggests that this feature actually
3481 * reduces performance on BDW, but it has only been observed to help so
3482 * far. Sampling fast-cleared blocks on BDW must also be handled with care
3483 * (see depth_stencil_attachment_compute_aux_usage() for more info).
3484 */
3485 if (devinfo->gen != 8 && !devinfo->has_sample_with_hiz)
3486 return false;
3487
3488 return image->samples == 1;
3489 }
3490
3491 static inline bool
3492 anv_image_plane_uses_aux_map(const struct anv_device *device,
3493 const struct anv_image *image,
3494 uint32_t plane)
3495 {
3496 return device->info.has_aux_map &&
3497 isl_aux_usage_has_ccs(image->planes[plane].aux_usage);
3498 }
3499
3500 void
3501 anv_cmd_buffer_mark_image_written(struct anv_cmd_buffer *cmd_buffer,
3502 const struct anv_image *image,
3503 VkImageAspectFlagBits aspect,
3504 enum isl_aux_usage aux_usage,
3505 uint32_t level,
3506 uint32_t base_layer,
3507 uint32_t layer_count);
3508
3509 void
3510 anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer,
3511 const struct anv_image *image,
3512 VkImageAspectFlagBits aspect,
3513 enum isl_aux_usage aux_usage,
3514 enum isl_format format, struct isl_swizzle swizzle,
3515 uint32_t level, uint32_t base_layer, uint32_t layer_count,
3516 VkRect2D area, union isl_color_value clear_color);
3517 void
3518 anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer,
3519 const struct anv_image *image,
3520 VkImageAspectFlags aspects,
3521 enum isl_aux_usage depth_aux_usage,
3522 uint32_t level,
3523 uint32_t base_layer, uint32_t layer_count,
3524 VkRect2D area,
3525 float depth_value, uint8_t stencil_value);
3526 void
3527 anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer,
3528 const struct anv_image *src_image,
3529 enum isl_aux_usage src_aux_usage,
3530 uint32_t src_level, uint32_t src_base_layer,
3531 const struct anv_image *dst_image,
3532 enum isl_aux_usage dst_aux_usage,
3533 uint32_t dst_level, uint32_t dst_base_layer,
3534 VkImageAspectFlagBits aspect,
3535 uint32_t src_x, uint32_t src_y,
3536 uint32_t dst_x, uint32_t dst_y,
3537 uint32_t width, uint32_t height,
3538 uint32_t layer_count,
3539 enum blorp_filter filter);
3540 void
3541 anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer,
3542 const struct anv_image *image,
3543 VkImageAspectFlagBits aspect, uint32_t level,
3544 uint32_t base_layer, uint32_t layer_count,
3545 enum isl_aux_op hiz_op);
3546 void
3547 anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer,
3548 const struct anv_image *image,
3549 VkImageAspectFlags aspects,
3550 uint32_t level,
3551 uint32_t base_layer, uint32_t layer_count,
3552 VkRect2D area, uint8_t stencil_value);
3553 void
3554 anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer,
3555 const struct anv_image *image,
3556 enum isl_format format,
3557 VkImageAspectFlagBits aspect,
3558 uint32_t base_layer, uint32_t layer_count,
3559 enum isl_aux_op mcs_op, union isl_color_value *clear_value,
3560 bool predicate);
3561 void
3562 anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer,
3563 const struct anv_image *image,
3564 enum isl_format format,
3565 VkImageAspectFlagBits aspect, uint32_t level,
3566 uint32_t base_layer, uint32_t layer_count,
3567 enum isl_aux_op ccs_op, union isl_color_value *clear_value,
3568 bool predicate);
3569
3570 void
3571 anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer,
3572 const struct anv_image *image,
3573 VkImageAspectFlagBits aspect,
3574 uint32_t base_level, uint32_t level_count,
3575 uint32_t base_layer, uint32_t layer_count);
3576
3577 enum isl_aux_usage
3578 anv_layout_to_aux_usage(const struct gen_device_info * const devinfo,
3579 const struct anv_image *image,
3580 const VkImageAspectFlagBits aspect,
3581 const VkImageLayout layout);
3582
3583 enum anv_fast_clear_type
3584 anv_layout_to_fast_clear_type(const struct gen_device_info * const devinfo,
3585 const struct anv_image * const image,
3586 const VkImageAspectFlagBits aspect,
3587 const VkImageLayout layout);
3588
3589 /* This is defined as a macro so that it works for both
3590 * VkImageSubresourceRange and VkImageSubresourceLayers
3591 */
3592 #define anv_get_layerCount(_image, _range) \
3593 ((_range)->layerCount == VK_REMAINING_ARRAY_LAYERS ? \
3594 (_image)->array_size - (_range)->baseArrayLayer : (_range)->layerCount)
3595
3596 static inline uint32_t
3597 anv_get_levelCount(const struct anv_image *image,
3598 const VkImageSubresourceRange *range)
3599 {
3600 return range->levelCount == VK_REMAINING_MIP_LEVELS ?
3601 image->levels - range->baseMipLevel : range->levelCount;
3602 }
3603
3604 static inline VkImageAspectFlags
3605 anv_image_expand_aspects(const struct anv_image *image,
3606 VkImageAspectFlags aspects)
3607 {
3608 /* If the underlying image has color plane aspects and
3609 * VK_IMAGE_ASPECT_COLOR_BIT has been requested, then return the aspects of
3610 * the underlying image. */
3611 if ((image->aspects & VK_IMAGE_ASPECT_PLANES_BITS_ANV) != 0 &&
3612 aspects == VK_IMAGE_ASPECT_COLOR_BIT)
3613 return image->aspects;
3614
3615 return aspects;
3616 }
3617
3618 static inline bool
3619 anv_image_aspects_compatible(VkImageAspectFlags aspects1,
3620 VkImageAspectFlags aspects2)
3621 {
3622 if (aspects1 == aspects2)
3623 return true;
3624
3625 /* Only 1 color aspects are compatibles. */
3626 if ((aspects1 & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) != 0 &&
3627 (aspects2 & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) != 0 &&
3628 util_bitcount(aspects1) == util_bitcount(aspects2))
3629 return true;
3630
3631 return false;
3632 }
3633
3634 struct anv_image_view {
3635 const struct anv_image *image; /**< VkImageViewCreateInfo::image */
3636
3637 VkImageAspectFlags aspect_mask;
3638 VkFormat vk_format;
3639 VkExtent3D extent; /**< Extent of VkImageViewCreateInfo::baseMipLevel. */
3640
3641 unsigned n_planes;
3642 struct {
3643 uint32_t image_plane;
3644
3645 struct isl_view isl;
3646
3647 /**
3648 * RENDER_SURFACE_STATE when using image as a sampler surface with an
3649 * image layout of SHADER_READ_ONLY_OPTIMAL or
3650 * DEPTH_STENCIL_READ_ONLY_OPTIMAL.
3651 */
3652 struct anv_surface_state optimal_sampler_surface_state;
3653
3654 /**
3655 * RENDER_SURFACE_STATE when using image as a sampler surface with an
3656 * image layout of GENERAL.
3657 */
3658 struct anv_surface_state general_sampler_surface_state;
3659
3660 /**
3661 * RENDER_SURFACE_STATE when using image as a storage image. Separate
3662 * states for write-only and readable, using the real format for
3663 * write-only and the lowered format for readable.
3664 */
3665 struct anv_surface_state storage_surface_state;
3666 struct anv_surface_state writeonly_storage_surface_state;
3667
3668 struct brw_image_param storage_image_param;
3669 } planes[3];
3670 };
3671
3672 enum anv_image_view_state_flags {
3673 ANV_IMAGE_VIEW_STATE_STORAGE_WRITE_ONLY = (1 << 0),
3674 ANV_IMAGE_VIEW_STATE_TEXTURE_OPTIMAL = (1 << 1),
3675 };
3676
3677 void anv_image_fill_surface_state(struct anv_device *device,
3678 const struct anv_image *image,
3679 VkImageAspectFlagBits aspect,
3680 const struct isl_view *view,
3681 isl_surf_usage_flags_t view_usage,
3682 enum isl_aux_usage aux_usage,
3683 const union isl_color_value *clear_color,
3684 enum anv_image_view_state_flags flags,
3685 struct anv_surface_state *state_inout,
3686 struct brw_image_param *image_param_out);
3687
3688 struct anv_image_create_info {
3689 const VkImageCreateInfo *vk_info;
3690
3691 /** An opt-in bitmask which filters an ISL-mapping of the Vulkan tiling. */
3692 isl_tiling_flags_t isl_tiling_flags;
3693
3694 /** These flags will be added to any derived from VkImageCreateInfo. */
3695 isl_surf_usage_flags_t isl_extra_usage_flags;
3696
3697 uint32_t stride;
3698 bool external_format;
3699 };
3700
3701 VkResult anv_image_create(VkDevice _device,
3702 const struct anv_image_create_info *info,
3703 const VkAllocationCallbacks* alloc,
3704 VkImage *pImage);
3705
3706 const struct anv_surface *
3707 anv_image_get_surface_for_aspect_mask(const struct anv_image *image,
3708 VkImageAspectFlags aspect_mask);
3709
3710 enum isl_format
3711 anv_isl_format_for_descriptor_type(VkDescriptorType type);
3712
3713 static inline struct VkExtent3D
3714 anv_sanitize_image_extent(const VkImageType imageType,
3715 const struct VkExtent3D imageExtent)
3716 {
3717 switch (imageType) {
3718 case VK_IMAGE_TYPE_1D:
3719 return (VkExtent3D) { imageExtent.width, 1, 1 };
3720 case VK_IMAGE_TYPE_2D:
3721 return (VkExtent3D) { imageExtent.width, imageExtent.height, 1 };
3722 case VK_IMAGE_TYPE_3D:
3723 return imageExtent;
3724 default:
3725 unreachable("invalid image type");
3726 }
3727 }
3728
3729 static inline struct VkOffset3D
3730 anv_sanitize_image_offset(const VkImageType imageType,
3731 const struct VkOffset3D imageOffset)
3732 {
3733 switch (imageType) {
3734 case VK_IMAGE_TYPE_1D:
3735 return (VkOffset3D) { imageOffset.x, 0, 0 };
3736 case VK_IMAGE_TYPE_2D:
3737 return (VkOffset3D) { imageOffset.x, imageOffset.y, 0 };
3738 case VK_IMAGE_TYPE_3D:
3739 return imageOffset;
3740 default:
3741 unreachable("invalid image type");
3742 }
3743 }
3744
3745 VkFormatFeatureFlags
3746 anv_get_image_format_features(const struct gen_device_info *devinfo,
3747 VkFormat vk_format,
3748 const struct anv_format *anv_format,
3749 VkImageTiling vk_tiling);
3750
3751 void anv_fill_buffer_surface_state(struct anv_device *device,
3752 struct anv_state state,
3753 enum isl_format format,
3754 struct anv_address address,
3755 uint32_t range, uint32_t stride);
3756
3757 static inline void
3758 anv_clear_color_from_att_state(union isl_color_value *clear_color,
3759 const struct anv_attachment_state *att_state,
3760 const struct anv_image_view *iview)
3761 {
3762 const struct isl_format_layout *view_fmtl =
3763 isl_format_get_layout(iview->planes[0].isl.format);
3764
3765 #define COPY_CLEAR_COLOR_CHANNEL(c, i) \
3766 if (view_fmtl->channels.c.bits) \
3767 clear_color->u32[i] = att_state->clear_value.color.uint32[i]
3768
3769 COPY_CLEAR_COLOR_CHANNEL(r, 0);
3770 COPY_CLEAR_COLOR_CHANNEL(g, 1);
3771 COPY_CLEAR_COLOR_CHANNEL(b, 2);
3772 COPY_CLEAR_COLOR_CHANNEL(a, 3);
3773
3774 #undef COPY_CLEAR_COLOR_CHANNEL
3775 }
3776
3777
3778 struct anv_ycbcr_conversion {
3779 const struct anv_format * format;
3780 VkSamplerYcbcrModelConversion ycbcr_model;
3781 VkSamplerYcbcrRange ycbcr_range;
3782 VkComponentSwizzle mapping[4];
3783 VkChromaLocation chroma_offsets[2];
3784 VkFilter chroma_filter;
3785 bool chroma_reconstruction;
3786 };
3787
3788 struct anv_sampler {
3789 uint32_t state[3][4];
3790 uint32_t n_planes;
3791 struct anv_ycbcr_conversion *conversion;
3792
3793 /* Blob of sampler state data which is guaranteed to be 32-byte aligned
3794 * and with a 32-byte stride for use as bindless samplers.
3795 */
3796 struct anv_state bindless_state;
3797 };
3798
3799 struct anv_framebuffer {
3800 uint32_t width;
3801 uint32_t height;
3802 uint32_t layers;
3803
3804 uint32_t attachment_count;
3805 struct anv_image_view * attachments[0];
3806 };
3807
3808 struct anv_subpass_attachment {
3809 VkImageUsageFlagBits usage;
3810 uint32_t attachment;
3811 VkImageLayout layout;
3812
3813 /* Used only with attachment containing stencil data. */
3814 VkImageLayout stencil_layout;
3815 };
3816
3817 struct anv_subpass {
3818 uint32_t attachment_count;
3819
3820 /**
3821 * A pointer to all attachment references used in this subpass.
3822 * Only valid if ::attachment_count > 0.
3823 */
3824 struct anv_subpass_attachment * attachments;
3825 uint32_t input_count;
3826 struct anv_subpass_attachment * input_attachments;
3827 uint32_t color_count;
3828 struct anv_subpass_attachment * color_attachments;
3829 struct anv_subpass_attachment * resolve_attachments;
3830
3831 struct anv_subpass_attachment * depth_stencil_attachment;
3832 struct anv_subpass_attachment * ds_resolve_attachment;
3833 VkResolveModeFlagBitsKHR depth_resolve_mode;
3834 VkResolveModeFlagBitsKHR stencil_resolve_mode;
3835
3836 uint32_t view_mask;
3837
3838 /** Subpass has a depth/stencil self-dependency */
3839 bool has_ds_self_dep;
3840
3841 /** Subpass has at least one color resolve attachment */
3842 bool has_color_resolve;
3843 };
3844
3845 static inline unsigned
3846 anv_subpass_view_count(const struct anv_subpass *subpass)
3847 {
3848 return MAX2(1, util_bitcount(subpass->view_mask));
3849 }
3850
3851 struct anv_render_pass_attachment {
3852 /* TODO: Consider using VkAttachmentDescription instead of storing each of
3853 * its members individually.
3854 */
3855 VkFormat format;
3856 uint32_t samples;
3857 VkImageUsageFlags usage;
3858 VkAttachmentLoadOp load_op;
3859 VkAttachmentStoreOp store_op;
3860 VkAttachmentLoadOp stencil_load_op;
3861 VkImageLayout initial_layout;
3862 VkImageLayout final_layout;
3863 VkImageLayout first_subpass_layout;
3864
3865 VkImageLayout stencil_initial_layout;
3866 VkImageLayout stencil_final_layout;
3867
3868 /* The subpass id in which the attachment will be used last. */
3869 uint32_t last_subpass_idx;
3870 };
3871
3872 struct anv_render_pass {
3873 uint32_t attachment_count;
3874 uint32_t subpass_count;
3875 /* An array of subpass_count+1 flushes, one per subpass boundary */
3876 enum anv_pipe_bits * subpass_flushes;
3877 struct anv_render_pass_attachment * attachments;
3878 struct anv_subpass subpasses[0];
3879 };
3880
3881 #define ANV_PIPELINE_STATISTICS_MASK 0x000007ff
3882
3883 struct anv_query_pool {
3884 VkQueryType type;
3885 VkQueryPipelineStatisticFlags pipeline_statistics;
3886 /** Stride between slots, in bytes */
3887 uint32_t stride;
3888 /** Number of slots in this query pool */
3889 uint32_t slots;
3890 struct anv_bo * bo;
3891 };
3892
3893 int anv_get_instance_entrypoint_index(const char *name);
3894 int anv_get_device_entrypoint_index(const char *name);
3895 int anv_get_physical_device_entrypoint_index(const char *name);
3896
3897 const char *anv_get_instance_entry_name(int index);
3898 const char *anv_get_physical_device_entry_name(int index);
3899 const char *anv_get_device_entry_name(int index);
3900
3901 bool
3902 anv_instance_entrypoint_is_enabled(int index, uint32_t core_version,
3903 const struct anv_instance_extension_table *instance);
3904 bool
3905 anv_physical_device_entrypoint_is_enabled(int index, uint32_t core_version,
3906 const struct anv_instance_extension_table *instance);
3907 bool
3908 anv_device_entrypoint_is_enabled(int index, uint32_t core_version,
3909 const struct anv_instance_extension_table *instance,
3910 const struct anv_device_extension_table *device);
3911
3912 void *anv_lookup_entrypoint(const struct gen_device_info *devinfo,
3913 const char *name);
3914
3915 void anv_dump_image_to_ppm(struct anv_device *device,
3916 struct anv_image *image, unsigned miplevel,
3917 unsigned array_layer, VkImageAspectFlagBits aspect,
3918 const char *filename);
3919
3920 enum anv_dump_action {
3921 ANV_DUMP_FRAMEBUFFERS_BIT = 0x1,
3922 };
3923
3924 void anv_dump_start(struct anv_device *device, enum anv_dump_action actions);
3925 void anv_dump_finish(void);
3926
3927 void anv_dump_add_attachments(struct anv_cmd_buffer *cmd_buffer);
3928
3929 static inline uint32_t
3930 anv_get_subpass_id(const struct anv_cmd_state * const cmd_state)
3931 {
3932 /* This function must be called from within a subpass. */
3933 assert(cmd_state->pass && cmd_state->subpass);
3934
3935 const uint32_t subpass_id = cmd_state->subpass - cmd_state->pass->subpasses;
3936
3937 /* The id of this subpass shouldn't exceed the number of subpasses in this
3938 * render pass minus 1.
3939 */
3940 assert(subpass_id < cmd_state->pass->subpass_count);
3941 return subpass_id;
3942 }
3943
3944 struct gen_perf_config *anv_get_perf(const struct gen_device_info *devinfo, int fd);
3945 void anv_device_perf_init(struct anv_device *device);
3946
3947 #define ANV_DEFINE_HANDLE_CASTS(__anv_type, __VkType) \
3948 \
3949 static inline struct __anv_type * \
3950 __anv_type ## _from_handle(__VkType _handle) \
3951 { \
3952 return (struct __anv_type *) _handle; \
3953 } \
3954 \
3955 static inline __VkType \
3956 __anv_type ## _to_handle(struct __anv_type *_obj) \
3957 { \
3958 return (__VkType) _obj; \
3959 }
3960
3961 #define ANV_DEFINE_NONDISP_HANDLE_CASTS(__anv_type, __VkType) \
3962 \
3963 static inline struct __anv_type * \
3964 __anv_type ## _from_handle(__VkType _handle) \
3965 { \
3966 return (struct __anv_type *)(uintptr_t) _handle; \
3967 } \
3968 \
3969 static inline __VkType \
3970 __anv_type ## _to_handle(struct __anv_type *_obj) \
3971 { \
3972 return (__VkType)(uintptr_t) _obj; \
3973 }
3974
3975 #define ANV_FROM_HANDLE(__anv_type, __name, __handle) \
3976 struct __anv_type *__name = __anv_type ## _from_handle(__handle)
3977
3978 ANV_DEFINE_HANDLE_CASTS(anv_cmd_buffer, VkCommandBuffer)
3979 ANV_DEFINE_HANDLE_CASTS(anv_device, VkDevice)
3980 ANV_DEFINE_HANDLE_CASTS(anv_instance, VkInstance)
3981 ANV_DEFINE_HANDLE_CASTS(anv_physical_device, VkPhysicalDevice)
3982 ANV_DEFINE_HANDLE_CASTS(anv_queue, VkQueue)
3983
3984 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_cmd_pool, VkCommandPool)
3985 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_buffer, VkBuffer)
3986 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_buffer_view, VkBufferView)
3987 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_descriptor_pool, VkDescriptorPool)
3988 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_descriptor_set, VkDescriptorSet)
3989 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_descriptor_set_layout, VkDescriptorSetLayout)