anv: Flatten descriptor bindings in anv_nir_apply_pipeline_layout
[mesa.git] / src / intel / vulkan / anv_private.h
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #ifndef ANV_PRIVATE_H
25 #define ANV_PRIVATE_H
26
27 #include <stdlib.h>
28 #include <stdio.h>
29 #include <stdbool.h>
30 #include <pthread.h>
31 #include <assert.h>
32 #include <stdint.h>
33 #include "drm-uapi/i915_drm.h"
34
35 #ifdef HAVE_VALGRIND
36 #include <valgrind.h>
37 #include <memcheck.h>
38 #define VG(x) x
39 #ifndef NDEBUG
40 #define __gen_validate_value(x) VALGRIND_CHECK_MEM_IS_DEFINED(&(x), sizeof(x))
41 #endif
42 #else
43 #define VG(x) ((void)0)
44 #endif
45
46 #include "common/gen_clflush.h"
47 #include "common/gen_decoder.h"
48 #include "common/gen_gem.h"
49 #include "dev/gen_device_info.h"
50 #include "blorp/blorp.h"
51 #include "compiler/brw_compiler.h"
52 #include "util/bitset.h"
53 #include "util/macros.h"
54 #include "util/hash_table.h"
55 #include "util/list.h"
56 #include "util/sparse_array.h"
57 #include "util/u_atomic.h"
58 #include "util/u_vector.h"
59 #include "util/u_math.h"
60 #include "util/vma.h"
61 #include "util/xmlconfig.h"
62 #include "vk_alloc.h"
63 #include "vk_debug_report.h"
64
65 /* Pre-declarations needed for WSI entrypoints */
66 struct wl_surface;
67 struct wl_display;
68 typedef struct xcb_connection_t xcb_connection_t;
69 typedef uint32_t xcb_visualid_t;
70 typedef uint32_t xcb_window_t;
71
72 struct anv_batch;
73 struct anv_buffer;
74 struct anv_buffer_view;
75 struct anv_image_view;
76 struct anv_instance;
77
78 struct gen_aux_map_context;
79 struct gen_l3_config;
80 struct gen_perf_config;
81
82 #include <vulkan/vulkan.h>
83 #include <vulkan/vulkan_intel.h>
84 #include <vulkan/vk_icd.h>
85
86 #include "anv_android.h"
87 #include "anv_entrypoints.h"
88 #include "anv_extensions.h"
89 #include "isl/isl.h"
90
91 #include "dev/gen_debug.h"
92 #include "common/intel_log.h"
93 #include "wsi_common.h"
94
95 #define NSEC_PER_SEC 1000000000ull
96
97 /* anv Virtual Memory Layout
98 * =========================
99 *
100 * When the anv driver is determining the virtual graphics addresses of memory
101 * objects itself using the softpin mechanism, the following memory ranges
102 * will be used.
103 *
104 * Three special considerations to notice:
105 *
106 * (1) the dynamic state pool is located within the same 4 GiB as the low
107 * heap. This is to work around a VF cache issue described in a comment in
108 * anv_physical_device_init_heaps.
109 *
110 * (2) the binding table pool is located at lower addresses than the surface
111 * state pool, within a 4 GiB range. This allows surface state base addresses
112 * to cover both binding tables (16 bit offsets) and surface states (32 bit
113 * offsets).
114 *
115 * (3) the last 4 GiB of the address space is withheld from the high
116 * heap. Various hardware units will read past the end of an object for
117 * various reasons. This healthy margin prevents reads from wrapping around
118 * 48-bit addresses.
119 */
120 #define LOW_HEAP_MIN_ADDRESS 0x000000001000ULL /* 4 KiB */
121 #define LOW_HEAP_MAX_ADDRESS 0x0000bfffffffULL
122 #define DYNAMIC_STATE_POOL_MIN_ADDRESS 0x0000c0000000ULL /* 3 GiB */
123 #define DYNAMIC_STATE_POOL_MAX_ADDRESS 0x0000ffffffffULL
124 #define BINDING_TABLE_POOL_MIN_ADDRESS 0x000100000000ULL /* 4 GiB */
125 #define BINDING_TABLE_POOL_MAX_ADDRESS 0x00013fffffffULL
126 #define SURFACE_STATE_POOL_MIN_ADDRESS 0x000140000000ULL /* 5 GiB */
127 #define SURFACE_STATE_POOL_MAX_ADDRESS 0x00017fffffffULL
128 #define INSTRUCTION_STATE_POOL_MIN_ADDRESS 0x000180000000ULL /* 6 GiB */
129 #define INSTRUCTION_STATE_POOL_MAX_ADDRESS 0x0001bfffffffULL
130 #define HIGH_HEAP_MIN_ADDRESS 0x0001c0000000ULL /* 7 GiB */
131
132 #define LOW_HEAP_SIZE \
133 (LOW_HEAP_MAX_ADDRESS - LOW_HEAP_MIN_ADDRESS + 1)
134 #define DYNAMIC_STATE_POOL_SIZE \
135 (DYNAMIC_STATE_POOL_MAX_ADDRESS - DYNAMIC_STATE_POOL_MIN_ADDRESS + 1)
136 #define BINDING_TABLE_POOL_SIZE \
137 (BINDING_TABLE_POOL_MAX_ADDRESS - BINDING_TABLE_POOL_MIN_ADDRESS + 1)
138 #define SURFACE_STATE_POOL_SIZE \
139 (SURFACE_STATE_POOL_MAX_ADDRESS - SURFACE_STATE_POOL_MIN_ADDRESS + 1)
140 #define INSTRUCTION_STATE_POOL_SIZE \
141 (INSTRUCTION_STATE_POOL_MAX_ADDRESS - INSTRUCTION_STATE_POOL_MIN_ADDRESS + 1)
142
143 /* Allowing different clear colors requires us to perform a depth resolve at
144 * the end of certain render passes. This is because while slow clears store
145 * the clear color in the HiZ buffer, fast clears (without a resolve) don't.
146 * See the PRMs for examples describing when additional resolves would be
147 * necessary. To enable fast clears without requiring extra resolves, we set
148 * the clear value to a globally-defined one. We could allow different values
149 * if the user doesn't expect coherent data during or after a render passes
150 * (VK_ATTACHMENT_STORE_OP_DONT_CARE), but such users (aside from the CTS)
151 * don't seem to exist yet. In almost all Vulkan applications tested thus far,
152 * 1.0f seems to be the only value used. The only application that doesn't set
153 * this value does so through the usage of an seemingly uninitialized clear
154 * value.
155 */
156 #define ANV_HZ_FC_VAL 1.0f
157
158 #define MAX_VBS 28
159 #define MAX_XFB_BUFFERS 4
160 #define MAX_XFB_STREAMS 4
161 #define MAX_SETS 8
162 #define MAX_RTS 8
163 #define MAX_VIEWPORTS 16
164 #define MAX_SCISSORS 16
165 #define MAX_PUSH_CONSTANTS_SIZE 128
166 #define MAX_DYNAMIC_BUFFERS 16
167 #define MAX_IMAGES 64
168 #define MAX_PUSH_DESCRIPTORS 32 /* Minimum requirement */
169 #define MAX_INLINE_UNIFORM_BLOCK_SIZE 4096
170 #define MAX_INLINE_UNIFORM_BLOCK_DESCRIPTORS 32
171
172 /* From the Skylake PRM Vol. 7 "Binding Table Surface State Model":
173 *
174 * "The surface state model is used when a Binding Table Index (specified
175 * in the message descriptor) of less than 240 is specified. In this model,
176 * the Binding Table Index is used to index into the binding table, and the
177 * binding table entry contains a pointer to the SURFACE_STATE."
178 *
179 * Binding table values above 240 are used for various things in the hardware
180 * such as stateless, stateless with incoherent cache, SLM, and bindless.
181 */
182 #define MAX_BINDING_TABLE_SIZE 240
183
184 /* The kernel relocation API has a limitation of a 32-bit delta value
185 * applied to the address before it is written which, in spite of it being
186 * unsigned, is treated as signed . Because of the way that this maps to
187 * the Vulkan API, we cannot handle an offset into a buffer that does not
188 * fit into a signed 32 bits. The only mechanism we have for dealing with
189 * this at the moment is to limit all VkDeviceMemory objects to a maximum
190 * of 2GB each. The Vulkan spec allows us to do this:
191 *
192 * "Some platforms may have a limit on the maximum size of a single
193 * allocation. For example, certain systems may fail to create
194 * allocations with a size greater than or equal to 4GB. Such a limit is
195 * implementation-dependent, and if such a failure occurs then the error
196 * VK_ERROR_OUT_OF_DEVICE_MEMORY should be returned."
197 *
198 * We don't use vk_error here because it's not an error so much as an
199 * indication to the application that the allocation is too large.
200 */
201 #define MAX_MEMORY_ALLOCATION_SIZE (1ull << 31)
202
203 #define ANV_SVGS_VB_INDEX MAX_VBS
204 #define ANV_DRAWID_VB_INDEX (MAX_VBS + 1)
205
206 /* We reserve this MI ALU register for the purpose of handling predication.
207 * Other code which uses the MI ALU should leave it alone.
208 */
209 #define ANV_PREDICATE_RESULT_REG 0x2678 /* MI_ALU_REG15 */
210
211 /* For gen12 we set the streamout buffers using 4 separate commands
212 * (3DSTATE_SO_BUFFER_INDEX_*) instead of 3DSTATE_SO_BUFFER. However the layout
213 * of the 3DSTATE_SO_BUFFER_INDEX_* commands is identical to that of
214 * 3DSTATE_SO_BUFFER apart from the SOBufferIndex field, so for now we use the
215 * 3DSTATE_SO_BUFFER command, but change the 3DCommandSubOpcode.
216 * SO_BUFFER_INDEX_0_CMD is actually the 3DCommandSubOpcode for
217 * 3DSTATE_SO_BUFFER_INDEX_0.
218 */
219 #define SO_BUFFER_INDEX_0_CMD 0x60
220 #define anv_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
221
222 static inline uint32_t
223 align_down_npot_u32(uint32_t v, uint32_t a)
224 {
225 return v - (v % a);
226 }
227
228 static inline uint32_t
229 align_u32(uint32_t v, uint32_t a)
230 {
231 assert(a != 0 && a == (a & -a));
232 return (v + a - 1) & ~(a - 1);
233 }
234
235 static inline uint64_t
236 align_u64(uint64_t v, uint64_t a)
237 {
238 assert(a != 0 && a == (a & -a));
239 return (v + a - 1) & ~(a - 1);
240 }
241
242 static inline int32_t
243 align_i32(int32_t v, int32_t a)
244 {
245 assert(a != 0 && a == (a & -a));
246 return (v + a - 1) & ~(a - 1);
247 }
248
249 /** Alignment must be a power of 2. */
250 static inline bool
251 anv_is_aligned(uintmax_t n, uintmax_t a)
252 {
253 assert(a == (a & -a));
254 return (n & (a - 1)) == 0;
255 }
256
257 static inline uint32_t
258 anv_minify(uint32_t n, uint32_t levels)
259 {
260 if (unlikely(n == 0))
261 return 0;
262 else
263 return MAX2(n >> levels, 1);
264 }
265
266 static inline float
267 anv_clamp_f(float f, float min, float max)
268 {
269 assert(min < max);
270
271 if (f > max)
272 return max;
273 else if (f < min)
274 return min;
275 else
276 return f;
277 }
278
279 static inline bool
280 anv_clear_mask(uint32_t *inout_mask, uint32_t clear_mask)
281 {
282 if (*inout_mask & clear_mask) {
283 *inout_mask &= ~clear_mask;
284 return true;
285 } else {
286 return false;
287 }
288 }
289
290 static inline union isl_color_value
291 vk_to_isl_color(VkClearColorValue color)
292 {
293 return (union isl_color_value) {
294 .u32 = {
295 color.uint32[0],
296 color.uint32[1],
297 color.uint32[2],
298 color.uint32[3],
299 },
300 };
301 }
302
303 static inline void *anv_unpack_ptr(uintptr_t ptr, int bits, int *flags)
304 {
305 uintptr_t mask = (1ull << bits) - 1;
306 *flags = ptr & mask;
307 return (void *) (ptr & ~mask);
308 }
309
310 static inline uintptr_t anv_pack_ptr(void *ptr, int bits, int flags)
311 {
312 uintptr_t value = (uintptr_t) ptr;
313 uintptr_t mask = (1ull << bits) - 1;
314 return value | (mask & flags);
315 }
316
317 #define for_each_bit(b, dword) \
318 for (uint32_t __dword = (dword); \
319 (b) = __builtin_ffs(__dword) - 1, __dword; \
320 __dword &= ~(1 << (b)))
321
322 #define typed_memcpy(dest, src, count) ({ \
323 STATIC_ASSERT(sizeof(*src) == sizeof(*dest)); \
324 memcpy((dest), (src), (count) * sizeof(*(src))); \
325 })
326
327 /* Mapping from anv object to VkDebugReportObjectTypeEXT. New types need
328 * to be added here in order to utilize mapping in debug/error/perf macros.
329 */
330 #define REPORT_OBJECT_TYPE(o) \
331 __builtin_choose_expr ( \
332 __builtin_types_compatible_p (__typeof (o), struct anv_instance*), \
333 VK_DEBUG_REPORT_OBJECT_TYPE_INSTANCE_EXT, \
334 __builtin_choose_expr ( \
335 __builtin_types_compatible_p (__typeof (o), struct anv_physical_device*), \
336 VK_DEBUG_REPORT_OBJECT_TYPE_PHYSICAL_DEVICE_EXT, \
337 __builtin_choose_expr ( \
338 __builtin_types_compatible_p (__typeof (o), struct anv_device*), \
339 VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT, \
340 __builtin_choose_expr ( \
341 __builtin_types_compatible_p (__typeof (o), const struct anv_device*), \
342 VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT, \
343 __builtin_choose_expr ( \
344 __builtin_types_compatible_p (__typeof (o), struct anv_queue*), \
345 VK_DEBUG_REPORT_OBJECT_TYPE_QUEUE_EXT, \
346 __builtin_choose_expr ( \
347 __builtin_types_compatible_p (__typeof (o), struct anv_semaphore*), \
348 VK_DEBUG_REPORT_OBJECT_TYPE_SEMAPHORE_EXT, \
349 __builtin_choose_expr ( \
350 __builtin_types_compatible_p (__typeof (o), struct anv_cmd_buffer*), \
351 VK_DEBUG_REPORT_OBJECT_TYPE_COMMAND_BUFFER_EXT, \
352 __builtin_choose_expr ( \
353 __builtin_types_compatible_p (__typeof (o), struct anv_fence*), \
354 VK_DEBUG_REPORT_OBJECT_TYPE_FENCE_EXT, \
355 __builtin_choose_expr ( \
356 __builtin_types_compatible_p (__typeof (o), struct anv_device_memory*), \
357 VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_MEMORY_EXT, \
358 __builtin_choose_expr ( \
359 __builtin_types_compatible_p (__typeof (o), struct anv_buffer*), \
360 VK_DEBUG_REPORT_OBJECT_TYPE_BUFFER_EXT, \
361 __builtin_choose_expr ( \
362 __builtin_types_compatible_p (__typeof (o), struct anv_image*), \
363 VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_EXT, \
364 __builtin_choose_expr ( \
365 __builtin_types_compatible_p (__typeof (o), const struct anv_image*), \
366 VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_EXT, \
367 __builtin_choose_expr ( \
368 __builtin_types_compatible_p (__typeof (o), struct anv_event*), \
369 VK_DEBUG_REPORT_OBJECT_TYPE_EVENT_EXT, \
370 __builtin_choose_expr ( \
371 __builtin_types_compatible_p (__typeof (o), struct anv_query_pool*), \
372 VK_DEBUG_REPORT_OBJECT_TYPE_QUERY_POOL_EXT, \
373 __builtin_choose_expr ( \
374 __builtin_types_compatible_p (__typeof (o), struct anv_buffer_view*), \
375 VK_DEBUG_REPORT_OBJECT_TYPE_BUFFER_VIEW_EXT, \
376 __builtin_choose_expr ( \
377 __builtin_types_compatible_p (__typeof (o), struct anv_image_view*), \
378 VK_DEBUG_REPORT_OBJECT_TYPE_IMAGE_VIEW_EXT, \
379 __builtin_choose_expr ( \
380 __builtin_types_compatible_p (__typeof (o), struct anv_shader_module*), \
381 VK_DEBUG_REPORT_OBJECT_TYPE_SHADER_MODULE_EXT, \
382 __builtin_choose_expr ( \
383 __builtin_types_compatible_p (__typeof (o), struct anv_pipeline_cache*), \
384 VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_CACHE_EXT, \
385 __builtin_choose_expr ( \
386 __builtin_types_compatible_p (__typeof (o), struct anv_pipeline_layout*), \
387 VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_LAYOUT_EXT, \
388 __builtin_choose_expr ( \
389 __builtin_types_compatible_p (__typeof (o), struct anv_render_pass*), \
390 VK_DEBUG_REPORT_OBJECT_TYPE_RENDER_PASS_EXT, \
391 __builtin_choose_expr ( \
392 __builtin_types_compatible_p (__typeof (o), struct anv_pipeline*), \
393 VK_DEBUG_REPORT_OBJECT_TYPE_PIPELINE_EXT, \
394 __builtin_choose_expr ( \
395 __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_set_layout*), \
396 VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT_EXT, \
397 __builtin_choose_expr ( \
398 __builtin_types_compatible_p (__typeof (o), struct anv_sampler*), \
399 VK_DEBUG_REPORT_OBJECT_TYPE_SAMPLER_EXT, \
400 __builtin_choose_expr ( \
401 __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_pool*), \
402 VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_POOL_EXT, \
403 __builtin_choose_expr ( \
404 __builtin_types_compatible_p (__typeof (o), struct anv_descriptor_set*), \
405 VK_DEBUG_REPORT_OBJECT_TYPE_DESCRIPTOR_SET_EXT, \
406 __builtin_choose_expr ( \
407 __builtin_types_compatible_p (__typeof (o), struct anv_framebuffer*), \
408 VK_DEBUG_REPORT_OBJECT_TYPE_FRAMEBUFFER_EXT, \
409 __builtin_choose_expr ( \
410 __builtin_types_compatible_p (__typeof (o), struct anv_cmd_pool*), \
411 VK_DEBUG_REPORT_OBJECT_TYPE_COMMAND_POOL_EXT, \
412 __builtin_choose_expr ( \
413 __builtin_types_compatible_p (__typeof (o), struct anv_surface*), \
414 VK_DEBUG_REPORT_OBJECT_TYPE_SURFACE_KHR_EXT, \
415 __builtin_choose_expr ( \
416 __builtin_types_compatible_p (__typeof (o), struct wsi_swapchain*), \
417 VK_DEBUG_REPORT_OBJECT_TYPE_SWAPCHAIN_KHR_EXT, \
418 __builtin_choose_expr ( \
419 __builtin_types_compatible_p (__typeof (o), struct vk_debug_callback*), \
420 VK_DEBUG_REPORT_OBJECT_TYPE_DEBUG_REPORT_CALLBACK_EXT_EXT, \
421 __builtin_choose_expr ( \
422 __builtin_types_compatible_p (__typeof (o), void*), \
423 VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT, \
424 /* The void expression results in a compile-time error \
425 when assigning the result to something. */ \
426 (void)0)))))))))))))))))))))))))))))))
427
428 /* Whenever we generate an error, pass it through this function. Useful for
429 * debugging, where we can break on it. Only call at error site, not when
430 * propagating errors. Might be useful to plug in a stack trace here.
431 */
432
433 VkResult __vk_errorv(struct anv_instance *instance, const void *object,
434 VkDebugReportObjectTypeEXT type, VkResult error,
435 const char *file, int line, const char *format,
436 va_list args);
437
438 VkResult __vk_errorf(struct anv_instance *instance, const void *object,
439 VkDebugReportObjectTypeEXT type, VkResult error,
440 const char *file, int line, const char *format, ...)
441 anv_printflike(7, 8);
442
443 #ifdef DEBUG
444 #define vk_error(error) __vk_errorf(NULL, NULL,\
445 VK_DEBUG_REPORT_OBJECT_TYPE_UNKNOWN_EXT,\
446 error, __FILE__, __LINE__, NULL)
447 #define vk_errorv(instance, obj, error, format, args)\
448 __vk_errorv(instance, obj, REPORT_OBJECT_TYPE(obj), error,\
449 __FILE__, __LINE__, format, args)
450 #define vk_errorf(instance, obj, error, format, ...)\
451 __vk_errorf(instance, obj, REPORT_OBJECT_TYPE(obj), error,\
452 __FILE__, __LINE__, format, ## __VA_ARGS__)
453 #else
454 #define vk_error(error) error
455 #define vk_errorf(instance, obj, error, format, ...) error
456 #endif
457
458 /**
459 * Warn on ignored extension structs.
460 *
461 * The Vulkan spec requires us to ignore unsupported or unknown structs in
462 * a pNext chain. In debug mode, emitting warnings for ignored structs may
463 * help us discover structs that we should not have ignored.
464 *
465 *
466 * From the Vulkan 1.0.38 spec:
467 *
468 * Any component of the implementation (the loader, any enabled layers,
469 * and drivers) must skip over, without processing (other than reading the
470 * sType and pNext members) any chained structures with sType values not
471 * defined by extensions supported by that component.
472 */
473 #define anv_debug_ignored_stype(sType) \
474 intel_logd("%s: ignored VkStructureType %u\n", __func__, (sType))
475
476 void __anv_perf_warn(struct anv_instance *instance, const void *object,
477 VkDebugReportObjectTypeEXT type, const char *file,
478 int line, const char *format, ...)
479 anv_printflike(6, 7);
480 void anv_loge(const char *format, ...) anv_printflike(1, 2);
481 void anv_loge_v(const char *format, va_list va);
482
483 /**
484 * Print a FINISHME message, including its source location.
485 */
486 #define anv_finishme(format, ...) \
487 do { \
488 static bool reported = false; \
489 if (!reported) { \
490 intel_logw("%s:%d: FINISHME: " format, __FILE__, __LINE__, \
491 ##__VA_ARGS__); \
492 reported = true; \
493 } \
494 } while (0)
495
496 /**
497 * Print a perf warning message. Set INTEL_DEBUG=perf to see these.
498 */
499 #define anv_perf_warn(instance, obj, format, ...) \
500 do { \
501 static bool reported = false; \
502 if (!reported && unlikely(INTEL_DEBUG & DEBUG_PERF)) { \
503 __anv_perf_warn(instance, obj, REPORT_OBJECT_TYPE(obj), __FILE__, __LINE__,\
504 format, ##__VA_ARGS__); \
505 reported = true; \
506 } \
507 } while (0)
508
509 /* A non-fatal assert. Useful for debugging. */
510 #ifdef DEBUG
511 #define anv_assert(x) ({ \
512 if (unlikely(!(x))) \
513 intel_loge("%s:%d ASSERT: %s", __FILE__, __LINE__, #x); \
514 })
515 #else
516 #define anv_assert(x)
517 #endif
518
519 /* A multi-pointer allocator
520 *
521 * When copying data structures from the user (such as a render pass), it's
522 * common to need to allocate data for a bunch of different things. Instead
523 * of doing several allocations and having to handle all of the error checking
524 * that entails, it can be easier to do a single allocation. This struct
525 * helps facilitate that. The intended usage looks like this:
526 *
527 * ANV_MULTIALLOC(ma)
528 * anv_multialloc_add(&ma, &main_ptr, 1);
529 * anv_multialloc_add(&ma, &substruct1, substruct1Count);
530 * anv_multialloc_add(&ma, &substruct2, substruct2Count);
531 *
532 * if (!anv_multialloc_alloc(&ma, pAllocator, VK_ALLOCATION_SCOPE_FOO))
533 * return vk_error(VK_ERROR_OUT_OF_HOST_MEORY);
534 */
535 struct anv_multialloc {
536 size_t size;
537 size_t align;
538
539 uint32_t ptr_count;
540 void **ptrs[8];
541 };
542
543 #define ANV_MULTIALLOC_INIT \
544 ((struct anv_multialloc) { 0, })
545
546 #define ANV_MULTIALLOC(_name) \
547 struct anv_multialloc _name = ANV_MULTIALLOC_INIT
548
549 __attribute__((always_inline))
550 static inline void
551 _anv_multialloc_add(struct anv_multialloc *ma,
552 void **ptr, size_t size, size_t align)
553 {
554 size_t offset = align_u64(ma->size, align);
555 ma->size = offset + size;
556 ma->align = MAX2(ma->align, align);
557
558 /* Store the offset in the pointer. */
559 *ptr = (void *)(uintptr_t)offset;
560
561 assert(ma->ptr_count < ARRAY_SIZE(ma->ptrs));
562 ma->ptrs[ma->ptr_count++] = ptr;
563 }
564
565 #define anv_multialloc_add_size(_ma, _ptr, _size) \
566 _anv_multialloc_add((_ma), (void **)(_ptr), (_size), __alignof__(**(_ptr)))
567
568 #define anv_multialloc_add(_ma, _ptr, _count) \
569 anv_multialloc_add_size(_ma, _ptr, (_count) * sizeof(**(_ptr)));
570
571 __attribute__((always_inline))
572 static inline void *
573 anv_multialloc_alloc(struct anv_multialloc *ma,
574 const VkAllocationCallbacks *alloc,
575 VkSystemAllocationScope scope)
576 {
577 void *ptr = vk_alloc(alloc, ma->size, ma->align, scope);
578 if (!ptr)
579 return NULL;
580
581 /* Fill out each of the pointers with their final value.
582 *
583 * for (uint32_t i = 0; i < ma->ptr_count; i++)
584 * *ma->ptrs[i] = ptr + (uintptr_t)*ma->ptrs[i];
585 *
586 * Unfortunately, even though ma->ptr_count is basically guaranteed to be a
587 * constant, GCC is incapable of figuring this out and unrolling the loop
588 * so we have to give it a little help.
589 */
590 STATIC_ASSERT(ARRAY_SIZE(ma->ptrs) == 8);
591 #define _ANV_MULTIALLOC_UPDATE_POINTER(_i) \
592 if ((_i) < ma->ptr_count) \
593 *ma->ptrs[_i] = ptr + (uintptr_t)*ma->ptrs[_i]
594 _ANV_MULTIALLOC_UPDATE_POINTER(0);
595 _ANV_MULTIALLOC_UPDATE_POINTER(1);
596 _ANV_MULTIALLOC_UPDATE_POINTER(2);
597 _ANV_MULTIALLOC_UPDATE_POINTER(3);
598 _ANV_MULTIALLOC_UPDATE_POINTER(4);
599 _ANV_MULTIALLOC_UPDATE_POINTER(5);
600 _ANV_MULTIALLOC_UPDATE_POINTER(6);
601 _ANV_MULTIALLOC_UPDATE_POINTER(7);
602 #undef _ANV_MULTIALLOC_UPDATE_POINTER
603
604 return ptr;
605 }
606
607 __attribute__((always_inline))
608 static inline void *
609 anv_multialloc_alloc2(struct anv_multialloc *ma,
610 const VkAllocationCallbacks *parent_alloc,
611 const VkAllocationCallbacks *alloc,
612 VkSystemAllocationScope scope)
613 {
614 return anv_multialloc_alloc(ma, alloc ? alloc : parent_alloc, scope);
615 }
616
617 struct anv_bo {
618 uint32_t gem_handle;
619
620 uint32_t refcount;
621
622 /* Index into the current validation list. This is used by the
623 * validation list building alrogithm to track which buffers are already
624 * in the validation list so that we can ensure uniqueness.
625 */
626 uint32_t index;
627
628 /* Index for use with util_sparse_array_free_list */
629 uint32_t free_index;
630
631 /* Last known offset. This value is provided by the kernel when we
632 * execbuf and is used as the presumed offset for the next bunch of
633 * relocations.
634 */
635 uint64_t offset;
636
637 uint64_t size;
638
639 /* Map for internally mapped BOs.
640 *
641 * If ANV_BO_WRAPPER is set in flags, map points to the wrapped BO.
642 */
643 void *map;
644
645 /** Flags to pass to the kernel through drm_i915_exec_object2::flags */
646 uint32_t flags;
647
648 /** True if this BO may be shared with other processes */
649 bool is_external:1;
650
651 /** True if this BO is a wrapper
652 *
653 * When set to true, none of the fields in this BO are meaningful except
654 * for anv_bo::is_wrapper and anv_bo::map which points to the actual BO.
655 * See also anv_bo_unwrap(). Wrapper BOs are not allowed when use_softpin
656 * is set in the physical device.
657 */
658 bool is_wrapper:1;
659
660 /** See also ANV_BO_ALLOC_FIXED_ADDRESS */
661 bool has_fixed_address:1;
662
663 /** True if this BO wraps a host pointer */
664 bool from_host_ptr:1;
665 };
666
667 static inline struct anv_bo *
668 anv_bo_unwrap(struct anv_bo *bo)
669 {
670 while (bo->is_wrapper)
671 bo = bo->map;
672 return bo;
673 }
674
675 /* Represents a lock-free linked list of "free" things. This is used by
676 * both the block pool and the state pools. Unfortunately, in order to
677 * solve the ABA problem, we can't use a single uint32_t head.
678 */
679 union anv_free_list {
680 struct {
681 uint32_t offset;
682
683 /* A simple count that is incremented every time the head changes. */
684 uint32_t count;
685 };
686 /* Make sure it's aligned to 64 bits. This will make atomic operations
687 * faster on 32 bit platforms.
688 */
689 uint64_t u64 __attribute__ ((aligned (8)));
690 };
691
692 #define ANV_FREE_LIST_EMPTY ((union anv_free_list) { { UINT32_MAX, 0 } })
693
694 struct anv_block_state {
695 union {
696 struct {
697 uint32_t next;
698 uint32_t end;
699 };
700 /* Make sure it's aligned to 64 bits. This will make atomic operations
701 * faster on 32 bit platforms.
702 */
703 uint64_t u64 __attribute__ ((aligned (8)));
704 };
705 };
706
707 #define anv_block_pool_foreach_bo(bo, pool) \
708 for (struct anv_bo **_pp_bo = (pool)->bos, *bo; \
709 _pp_bo != &(pool)->bos[(pool)->nbos] && (bo = *_pp_bo, true); \
710 _pp_bo++)
711
712 #define ANV_MAX_BLOCK_POOL_BOS 20
713
714 struct anv_block_pool {
715 struct anv_device *device;
716 bool use_softpin;
717
718 /* Wrapper BO for use in relocation lists. This BO is simply a wrapper
719 * around the actual BO so that we grow the pool after the wrapper BO has
720 * been put in a relocation list. This is only used in the non-softpin
721 * case.
722 */
723 struct anv_bo wrapper_bo;
724
725 struct anv_bo *bos[ANV_MAX_BLOCK_POOL_BOS];
726 struct anv_bo *bo;
727 uint32_t nbos;
728
729 uint64_t size;
730
731 /* The address where the start of the pool is pinned. The various bos that
732 * are created as the pool grows will have addresses in the range
733 * [start_address, start_address + BLOCK_POOL_MEMFD_SIZE).
734 */
735 uint64_t start_address;
736
737 /* The offset from the start of the bo to the "center" of the block
738 * pool. Pointers to allocated blocks are given by
739 * bo.map + center_bo_offset + offsets.
740 */
741 uint32_t center_bo_offset;
742
743 /* Current memory map of the block pool. This pointer may or may not
744 * point to the actual beginning of the block pool memory. If
745 * anv_block_pool_alloc_back has ever been called, then this pointer
746 * will point to the "center" position of the buffer and all offsets
747 * (negative or positive) given out by the block pool alloc functions
748 * will be valid relative to this pointer.
749 *
750 * In particular, map == bo.map + center_offset
751 *
752 * DO NOT access this pointer directly. Use anv_block_pool_map() instead,
753 * since it will handle the softpin case as well, where this points to NULL.
754 */
755 void *map;
756 int fd;
757
758 /**
759 * Array of mmaps and gem handles owned by the block pool, reclaimed when
760 * the block pool is destroyed.
761 */
762 struct u_vector mmap_cleanups;
763
764 struct anv_block_state state;
765
766 struct anv_block_state back_state;
767 };
768
769 /* Block pools are backed by a fixed-size 1GB memfd */
770 #define BLOCK_POOL_MEMFD_SIZE (1ul << 30)
771
772 /* The center of the block pool is also the middle of the memfd. This may
773 * change in the future if we decide differently for some reason.
774 */
775 #define BLOCK_POOL_MEMFD_CENTER (BLOCK_POOL_MEMFD_SIZE / 2)
776
777 static inline uint32_t
778 anv_block_pool_size(struct anv_block_pool *pool)
779 {
780 return pool->state.end + pool->back_state.end;
781 }
782
783 struct anv_state {
784 int32_t offset;
785 uint32_t alloc_size;
786 void *map;
787 uint32_t idx;
788 };
789
790 #define ANV_STATE_NULL ((struct anv_state) { .alloc_size = 0 })
791
792 struct anv_fixed_size_state_pool {
793 union anv_free_list free_list;
794 struct anv_block_state block;
795 };
796
797 #define ANV_MIN_STATE_SIZE_LOG2 6
798 #define ANV_MAX_STATE_SIZE_LOG2 21
799
800 #define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2 + 1)
801
802 struct anv_free_entry {
803 uint32_t next;
804 struct anv_state state;
805 };
806
807 struct anv_state_table {
808 struct anv_device *device;
809 int fd;
810 struct anv_free_entry *map;
811 uint32_t size;
812 struct anv_block_state state;
813 struct u_vector cleanups;
814 };
815
816 struct anv_state_pool {
817 struct anv_block_pool block_pool;
818
819 struct anv_state_table table;
820
821 /* The size of blocks which will be allocated from the block pool */
822 uint32_t block_size;
823
824 /** Free list for "back" allocations */
825 union anv_free_list back_alloc_free_list;
826
827 struct anv_fixed_size_state_pool buckets[ANV_STATE_BUCKETS];
828 };
829
830 struct anv_state_stream_block;
831
832 struct anv_state_stream {
833 struct anv_state_pool *state_pool;
834
835 /* The size of blocks to allocate from the state pool */
836 uint32_t block_size;
837
838 /* Current block we're allocating from */
839 struct anv_state block;
840
841 /* Offset into the current block at which to allocate the next state */
842 uint32_t next;
843
844 /* List of all blocks allocated from this pool */
845 struct anv_state_stream_block *block_list;
846 };
847
848 /* The block_pool functions exported for testing only. The block pool should
849 * only be used via a state pool (see below).
850 */
851 VkResult anv_block_pool_init(struct anv_block_pool *pool,
852 struct anv_device *device,
853 uint64_t start_address,
854 uint32_t initial_size);
855 void anv_block_pool_finish(struct anv_block_pool *pool);
856 int32_t anv_block_pool_alloc(struct anv_block_pool *pool,
857 uint32_t block_size, uint32_t *padding);
858 int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool,
859 uint32_t block_size);
860 void* anv_block_pool_map(struct anv_block_pool *pool, int32_t offset);
861
862 VkResult anv_state_pool_init(struct anv_state_pool *pool,
863 struct anv_device *device,
864 uint64_t start_address,
865 uint32_t block_size);
866 void anv_state_pool_finish(struct anv_state_pool *pool);
867 struct anv_state anv_state_pool_alloc(struct anv_state_pool *pool,
868 uint32_t state_size, uint32_t alignment);
869 struct anv_state anv_state_pool_alloc_back(struct anv_state_pool *pool);
870 void anv_state_pool_free(struct anv_state_pool *pool, struct anv_state state);
871 void anv_state_stream_init(struct anv_state_stream *stream,
872 struct anv_state_pool *state_pool,
873 uint32_t block_size);
874 void anv_state_stream_finish(struct anv_state_stream *stream);
875 struct anv_state anv_state_stream_alloc(struct anv_state_stream *stream,
876 uint32_t size, uint32_t alignment);
877
878 VkResult anv_state_table_init(struct anv_state_table *table,
879 struct anv_device *device,
880 uint32_t initial_entries);
881 void anv_state_table_finish(struct anv_state_table *table);
882 VkResult anv_state_table_add(struct anv_state_table *table, uint32_t *idx,
883 uint32_t count);
884 void anv_free_list_push(union anv_free_list *list,
885 struct anv_state_table *table,
886 uint32_t idx, uint32_t count);
887 struct anv_state* anv_free_list_pop(union anv_free_list *list,
888 struct anv_state_table *table);
889
890
891 static inline struct anv_state *
892 anv_state_table_get(struct anv_state_table *table, uint32_t idx)
893 {
894 return &table->map[idx].state;
895 }
896 /**
897 * Implements a pool of re-usable BOs. The interface is identical to that
898 * of block_pool except that each block is its own BO.
899 */
900 struct anv_bo_pool {
901 struct anv_device *device;
902
903 uint64_t bo_flags;
904
905 struct util_sparse_array_free_list free_list[16];
906 };
907
908 void anv_bo_pool_init(struct anv_bo_pool *pool, struct anv_device *device,
909 uint64_t bo_flags);
910 void anv_bo_pool_finish(struct anv_bo_pool *pool);
911 VkResult anv_bo_pool_alloc(struct anv_bo_pool *pool, uint32_t size,
912 struct anv_bo **bo_out);
913 void anv_bo_pool_free(struct anv_bo_pool *pool, struct anv_bo *bo);
914
915 struct anv_scratch_pool {
916 /* Indexed by Per-Thread Scratch Space number (the hardware value) and stage */
917 struct anv_bo *bos[16][MESA_SHADER_STAGES];
918 };
919
920 void anv_scratch_pool_init(struct anv_device *device,
921 struct anv_scratch_pool *pool);
922 void anv_scratch_pool_finish(struct anv_device *device,
923 struct anv_scratch_pool *pool);
924 struct anv_bo *anv_scratch_pool_alloc(struct anv_device *device,
925 struct anv_scratch_pool *pool,
926 gl_shader_stage stage,
927 unsigned per_thread_scratch);
928
929 /** Implements a BO cache that ensures a 1-1 mapping of GEM BOs to anv_bos */
930 struct anv_bo_cache {
931 struct util_sparse_array bo_map;
932 pthread_mutex_t mutex;
933 };
934
935 VkResult anv_bo_cache_init(struct anv_bo_cache *cache);
936 void anv_bo_cache_finish(struct anv_bo_cache *cache);
937
938 struct anv_memory_type {
939 /* Standard bits passed on to the client */
940 VkMemoryPropertyFlags propertyFlags;
941 uint32_t heapIndex;
942
943 /* Driver-internal book-keeping */
944 VkBufferUsageFlags valid_buffer_usage;
945 };
946
947 struct anv_memory_heap {
948 /* Standard bits passed on to the client */
949 VkDeviceSize size;
950 VkMemoryHeapFlags flags;
951
952 /* Driver-internal book-keeping */
953 uint64_t vma_start;
954 uint64_t vma_size;
955 bool supports_48bit_addresses;
956 VkDeviceSize used;
957 };
958
959 struct anv_physical_device {
960 VK_LOADER_DATA _loader_data;
961
962 struct anv_instance * instance;
963 uint32_t chipset_id;
964 bool no_hw;
965 char path[20];
966 const char * name;
967 struct {
968 uint16_t domain;
969 uint8_t bus;
970 uint8_t device;
971 uint8_t function;
972 } pci_info;
973 struct gen_device_info info;
974 /** Amount of "GPU memory" we want to advertise
975 *
976 * Clearly, this value is bogus since Intel is a UMA architecture. On
977 * gen7 platforms, we are limited by GTT size unless we want to implement
978 * fine-grained tracking and GTT splitting. On Broadwell and above we are
979 * practically unlimited. However, we will never report more than 3/4 of
980 * the total system ram to try and avoid running out of RAM.
981 */
982 bool supports_48bit_addresses;
983 struct brw_compiler * compiler;
984 struct isl_device isl_dev;
985 struct gen_perf_config * perf;
986 int cmd_parser_version;
987 bool has_exec_async;
988 bool has_exec_capture;
989 bool has_exec_fence;
990 bool has_syncobj;
991 bool has_syncobj_wait;
992 bool has_context_priority;
993 bool use_softpin;
994 bool has_context_isolation;
995 bool has_mem_available;
996 bool always_use_bindless;
997
998 /** True if we can access buffers using A64 messages */
999 bool has_a64_buffer_access;
1000 /** True if we can use bindless access for images */
1001 bool has_bindless_images;
1002 /** True if we can use bindless access for samplers */
1003 bool has_bindless_samplers;
1004
1005 struct anv_device_extension_table supported_extensions;
1006 struct anv_physical_device_dispatch_table dispatch;
1007
1008 uint32_t eu_total;
1009 uint32_t subslice_total;
1010
1011 struct {
1012 uint32_t type_count;
1013 struct anv_memory_type types[VK_MAX_MEMORY_TYPES];
1014 uint32_t heap_count;
1015 struct anv_memory_heap heaps[VK_MAX_MEMORY_HEAPS];
1016 } memory;
1017
1018 uint8_t driver_build_sha1[20];
1019 uint8_t pipeline_cache_uuid[VK_UUID_SIZE];
1020 uint8_t driver_uuid[VK_UUID_SIZE];
1021 uint8_t device_uuid[VK_UUID_SIZE];
1022
1023 struct disk_cache * disk_cache;
1024
1025 struct wsi_device wsi_device;
1026 int local_fd;
1027 int master_fd;
1028 };
1029
1030 struct anv_app_info {
1031 const char* app_name;
1032 uint32_t app_version;
1033 const char* engine_name;
1034 uint32_t engine_version;
1035 uint32_t api_version;
1036 };
1037
1038 struct anv_instance {
1039 VK_LOADER_DATA _loader_data;
1040
1041 VkAllocationCallbacks alloc;
1042
1043 struct anv_app_info app_info;
1044
1045 struct anv_instance_extension_table enabled_extensions;
1046 struct anv_instance_dispatch_table dispatch;
1047 struct anv_device_dispatch_table device_dispatch;
1048
1049 int physicalDeviceCount;
1050 struct anv_physical_device physicalDevice;
1051
1052 bool pipeline_cache_enabled;
1053
1054 struct vk_debug_report_instance debug_report_callbacks;
1055
1056 struct driOptionCache dri_options;
1057 struct driOptionCache available_dri_options;
1058 };
1059
1060 VkResult anv_init_wsi(struct anv_physical_device *physical_device);
1061 void anv_finish_wsi(struct anv_physical_device *physical_device);
1062
1063 uint32_t anv_physical_device_api_version(struct anv_physical_device *dev);
1064 bool anv_physical_device_extension_supported(struct anv_physical_device *dev,
1065 const char *name);
1066
1067 struct anv_queue_submit {
1068 struct anv_cmd_buffer * cmd_buffer;
1069
1070 uint32_t fence_count;
1071 uint32_t fence_array_length;
1072 struct drm_i915_gem_exec_fence * fences;
1073
1074 uint32_t temporary_semaphore_count;
1075 uint32_t temporary_semaphore_array_length;
1076 struct anv_semaphore_impl * temporary_semaphores;
1077
1078 /* Semaphores to be signaled with a SYNC_FD. */
1079 struct anv_semaphore ** sync_fd_semaphores;
1080 uint32_t sync_fd_semaphore_count;
1081 uint32_t sync_fd_semaphore_array_length;
1082
1083 /* Allocated only with non shareable timelines. */
1084 struct anv_timeline ** wait_timelines;
1085 uint32_t wait_timeline_count;
1086 uint32_t wait_timeline_array_length;
1087 uint64_t * wait_timeline_values;
1088
1089 struct anv_timeline ** signal_timelines;
1090 uint32_t signal_timeline_count;
1091 uint32_t signal_timeline_array_length;
1092 uint64_t * signal_timeline_values;
1093
1094 int in_fence;
1095 bool need_out_fence;
1096 int out_fence;
1097
1098 uint32_t fence_bo_count;
1099 uint32_t fence_bo_array_length;
1100 /* An array of struct anv_bo pointers with lower bit used as a flag to
1101 * signal we will wait on that BO (see anv_(un)pack_ptr).
1102 */
1103 uintptr_t * fence_bos;
1104
1105 const VkAllocationCallbacks * alloc;
1106 VkSystemAllocationScope alloc_scope;
1107
1108 struct anv_bo * simple_bo;
1109 uint32_t simple_bo_size;
1110
1111 struct list_head link;
1112 };
1113
1114 struct anv_queue {
1115 VK_LOADER_DATA _loader_data;
1116
1117 struct anv_device * device;
1118
1119 /*
1120 * A list of struct anv_queue_submit to be submitted to i915.
1121 */
1122 struct list_head queued_submits;
1123
1124 VkDeviceQueueCreateFlags flags;
1125 };
1126
1127 struct anv_pipeline_cache {
1128 struct anv_device * device;
1129 pthread_mutex_t mutex;
1130
1131 struct hash_table * nir_cache;
1132
1133 struct hash_table * cache;
1134 };
1135
1136 struct nir_xfb_info;
1137 struct anv_pipeline_bind_map;
1138
1139 void anv_pipeline_cache_init(struct anv_pipeline_cache *cache,
1140 struct anv_device *device,
1141 bool cache_enabled);
1142 void anv_pipeline_cache_finish(struct anv_pipeline_cache *cache);
1143
1144 struct anv_shader_bin *
1145 anv_pipeline_cache_search(struct anv_pipeline_cache *cache,
1146 const void *key, uint32_t key_size);
1147 struct anv_shader_bin *
1148 anv_pipeline_cache_upload_kernel(struct anv_pipeline_cache *cache,
1149 const void *key_data, uint32_t key_size,
1150 const void *kernel_data, uint32_t kernel_size,
1151 const void *constant_data,
1152 uint32_t constant_data_size,
1153 const struct brw_stage_prog_data *prog_data,
1154 uint32_t prog_data_size,
1155 const struct brw_compile_stats *stats,
1156 uint32_t num_stats,
1157 const struct nir_xfb_info *xfb_info,
1158 const struct anv_pipeline_bind_map *bind_map);
1159
1160 struct anv_shader_bin *
1161 anv_device_search_for_kernel(struct anv_device *device,
1162 struct anv_pipeline_cache *cache,
1163 const void *key_data, uint32_t key_size,
1164 bool *user_cache_bit);
1165
1166 struct anv_shader_bin *
1167 anv_device_upload_kernel(struct anv_device *device,
1168 struct anv_pipeline_cache *cache,
1169 const void *key_data, uint32_t key_size,
1170 const void *kernel_data, uint32_t kernel_size,
1171 const void *constant_data,
1172 uint32_t constant_data_size,
1173 const struct brw_stage_prog_data *prog_data,
1174 uint32_t prog_data_size,
1175 const struct brw_compile_stats *stats,
1176 uint32_t num_stats,
1177 const struct nir_xfb_info *xfb_info,
1178 const struct anv_pipeline_bind_map *bind_map);
1179
1180 struct nir_shader;
1181 struct nir_shader_compiler_options;
1182
1183 struct nir_shader *
1184 anv_device_search_for_nir(struct anv_device *device,
1185 struct anv_pipeline_cache *cache,
1186 const struct nir_shader_compiler_options *nir_options,
1187 unsigned char sha1_key[20],
1188 void *mem_ctx);
1189
1190 void
1191 anv_device_upload_nir(struct anv_device *device,
1192 struct anv_pipeline_cache *cache,
1193 const struct nir_shader *nir,
1194 unsigned char sha1_key[20]);
1195
1196 struct anv_device {
1197 VK_LOADER_DATA _loader_data;
1198
1199 VkAllocationCallbacks alloc;
1200
1201 struct anv_instance * instance;
1202 uint32_t chipset_id;
1203 bool no_hw;
1204 struct gen_device_info info;
1205 struct isl_device isl_dev;
1206 int context_id;
1207 int fd;
1208 bool can_chain_batches;
1209 bool robust_buffer_access;
1210 struct anv_device_extension_table enabled_extensions;
1211 struct anv_device_dispatch_table dispatch;
1212
1213 pthread_mutex_t vma_mutex;
1214 struct util_vma_heap vma_lo;
1215 struct util_vma_heap vma_hi;
1216 uint64_t vma_lo_available;
1217 uint64_t vma_hi_available;
1218
1219 /** List of all anv_device_memory objects */
1220 struct list_head memory_objects;
1221
1222 struct anv_bo_pool batch_bo_pool;
1223
1224 struct anv_bo_cache bo_cache;
1225
1226 struct anv_state_pool dynamic_state_pool;
1227 struct anv_state_pool instruction_state_pool;
1228 struct anv_state_pool binding_table_pool;
1229 struct anv_state_pool surface_state_pool;
1230
1231 struct anv_bo * workaround_bo;
1232 struct anv_bo * trivial_batch_bo;
1233 struct anv_bo * hiz_clear_bo;
1234
1235 struct anv_pipeline_cache default_pipeline_cache;
1236 struct blorp_context blorp;
1237
1238 struct anv_state border_colors;
1239
1240 struct anv_state slice_hash;
1241
1242 struct anv_queue queue;
1243
1244 struct anv_scratch_pool scratch_pool;
1245
1246 pthread_mutex_t mutex;
1247 pthread_cond_t queue_submit;
1248 int _lost;
1249
1250 struct gen_batch_decode_ctx decoder_ctx;
1251 /*
1252 * When decoding a anv_cmd_buffer, we might need to search for BOs through
1253 * the cmd_buffer's list.
1254 */
1255 struct anv_cmd_buffer *cmd_buffer_being_decoded;
1256
1257 int perf_fd; /* -1 if no opened */
1258 uint64_t perf_metric; /* 0 if unset */
1259
1260 struct gen_aux_map_context *aux_map_ctx;
1261 };
1262
1263 static inline struct anv_state_pool *
1264 anv_binding_table_pool(struct anv_device *device)
1265 {
1266 if (device->instance->physicalDevice.use_softpin)
1267 return &device->binding_table_pool;
1268 else
1269 return &device->surface_state_pool;
1270 }
1271
1272 static inline struct anv_state
1273 anv_binding_table_pool_alloc(struct anv_device *device) {
1274 if (device->instance->physicalDevice.use_softpin)
1275 return anv_state_pool_alloc(&device->binding_table_pool,
1276 device->binding_table_pool.block_size, 0);
1277 else
1278 return anv_state_pool_alloc_back(&device->surface_state_pool);
1279 }
1280
1281 static inline void
1282 anv_binding_table_pool_free(struct anv_device *device, struct anv_state state) {
1283 anv_state_pool_free(anv_binding_table_pool(device), state);
1284 }
1285
1286 static inline uint32_t
1287 anv_mocs_for_bo(const struct anv_device *device, const struct anv_bo *bo)
1288 {
1289 if (bo->is_external)
1290 return device->isl_dev.mocs.external;
1291 else
1292 return device->isl_dev.mocs.internal;
1293 }
1294
1295 void anv_device_init_blorp(struct anv_device *device);
1296 void anv_device_finish_blorp(struct anv_device *device);
1297
1298 void _anv_device_set_all_queue_lost(struct anv_device *device);
1299 VkResult _anv_device_set_lost(struct anv_device *device,
1300 const char *file, int line,
1301 const char *msg, ...)
1302 anv_printflike(4, 5);
1303 VkResult _anv_queue_set_lost(struct anv_queue *queue,
1304 const char *file, int line,
1305 const char *msg, ...)
1306 anv_printflike(4, 5);
1307 #define anv_device_set_lost(dev, ...) \
1308 _anv_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
1309 #define anv_queue_set_lost(queue, ...) \
1310 _anv_queue_set_lost(queue, __FILE__, __LINE__, __VA_ARGS__)
1311
1312 static inline bool
1313 anv_device_is_lost(struct anv_device *device)
1314 {
1315 return unlikely(p_atomic_read(&device->_lost));
1316 }
1317
1318 VkResult anv_device_query_status(struct anv_device *device);
1319
1320
1321 enum anv_bo_alloc_flags {
1322 /** Specifies that the BO must have a 32-bit address
1323 *
1324 * This is the opposite of EXEC_OBJECT_SUPPORTS_48B_ADDRESS.
1325 */
1326 ANV_BO_ALLOC_32BIT_ADDRESS = (1 << 0),
1327
1328 /** Specifies that the BO may be shared externally */
1329 ANV_BO_ALLOC_EXTERNAL = (1 << 1),
1330
1331 /** Specifies that the BO should be mapped */
1332 ANV_BO_ALLOC_MAPPED = (1 << 2),
1333
1334 /** Specifies that the BO should be snooped so we get coherency */
1335 ANV_BO_ALLOC_SNOOPED = (1 << 3),
1336
1337 /** Specifies that the BO should be captured in error states */
1338 ANV_BO_ALLOC_CAPTURE = (1 << 4),
1339
1340 /** Specifies that the BO will have an address assigned by the caller */
1341 ANV_BO_ALLOC_FIXED_ADDRESS = (1 << 5),
1342
1343 /** Enables implicit synchronization on the BO
1344 *
1345 * This is the opposite of EXEC_OBJECT_ASYNC.
1346 */
1347 ANV_BO_ALLOC_IMPLICIT_SYNC = (1 << 6),
1348
1349 /** Enables implicit synchronization on the BO
1350 *
1351 * This is equivalent to EXEC_OBJECT_WRITE.
1352 */
1353 ANV_BO_ALLOC_IMPLICIT_WRITE = (1 << 7),
1354 };
1355
1356 VkResult anv_device_alloc_bo(struct anv_device *device, uint64_t size,
1357 enum anv_bo_alloc_flags alloc_flags,
1358 struct anv_bo **bo);
1359 VkResult anv_device_import_bo_from_host_ptr(struct anv_device *device,
1360 void *host_ptr, uint32_t size,
1361 enum anv_bo_alloc_flags alloc_flags,
1362 struct anv_bo **bo_out);
1363 VkResult anv_device_import_bo(struct anv_device *device, int fd,
1364 enum anv_bo_alloc_flags alloc_flags,
1365 struct anv_bo **bo);
1366 VkResult anv_device_export_bo(struct anv_device *device,
1367 struct anv_bo *bo, int *fd_out);
1368 void anv_device_release_bo(struct anv_device *device,
1369 struct anv_bo *bo);
1370
1371 static inline struct anv_bo *
1372 anv_device_lookup_bo(struct anv_device *device, uint32_t gem_handle)
1373 {
1374 return util_sparse_array_get(&device->bo_cache.bo_map, gem_handle);
1375 }
1376
1377 VkResult anv_device_bo_busy(struct anv_device *device, struct anv_bo *bo);
1378 VkResult anv_device_wait(struct anv_device *device, struct anv_bo *bo,
1379 int64_t timeout);
1380
1381 VkResult anv_queue_init(struct anv_device *device, struct anv_queue *queue);
1382 void anv_queue_finish(struct anv_queue *queue);
1383
1384 VkResult anv_queue_execbuf_locked(struct anv_queue *queue, struct anv_queue_submit *submit);
1385 VkResult anv_queue_submit_simple_batch(struct anv_queue *queue,
1386 struct anv_batch *batch);
1387
1388 uint64_t anv_gettime_ns(void);
1389 uint64_t anv_get_absolute_timeout(uint64_t timeout);
1390
1391 void* anv_gem_mmap(struct anv_device *device,
1392 uint32_t gem_handle, uint64_t offset, uint64_t size, uint32_t flags);
1393 void anv_gem_munmap(void *p, uint64_t size);
1394 uint32_t anv_gem_create(struct anv_device *device, uint64_t size);
1395 void anv_gem_close(struct anv_device *device, uint32_t gem_handle);
1396 uint32_t anv_gem_userptr(struct anv_device *device, void *mem, size_t size);
1397 int anv_gem_busy(struct anv_device *device, uint32_t gem_handle);
1398 int anv_gem_wait(struct anv_device *device, uint32_t gem_handle, int64_t *timeout_ns);
1399 int anv_gem_execbuffer(struct anv_device *device,
1400 struct drm_i915_gem_execbuffer2 *execbuf);
1401 int anv_gem_set_tiling(struct anv_device *device, uint32_t gem_handle,
1402 uint32_t stride, uint32_t tiling);
1403 int anv_gem_create_context(struct anv_device *device);
1404 bool anv_gem_has_context_priority(int fd);
1405 int anv_gem_destroy_context(struct anv_device *device, int context);
1406 int anv_gem_set_context_param(int fd, int context, uint32_t param,
1407 uint64_t value);
1408 int anv_gem_get_context_param(int fd, int context, uint32_t param,
1409 uint64_t *value);
1410 int anv_gem_get_param(int fd, uint32_t param);
1411 int anv_gem_get_tiling(struct anv_device *device, uint32_t gem_handle);
1412 bool anv_gem_get_bit6_swizzle(int fd, uint32_t tiling);
1413 int anv_gem_get_aperture(int fd, uint64_t *size);
1414 int anv_gem_gpu_get_reset_stats(struct anv_device *device,
1415 uint32_t *active, uint32_t *pending);
1416 int anv_gem_handle_to_fd(struct anv_device *device, uint32_t gem_handle);
1417 int anv_gem_reg_read(struct anv_device *device,
1418 uint32_t offset, uint64_t *result);
1419 uint32_t anv_gem_fd_to_handle(struct anv_device *device, int fd);
1420 int anv_gem_set_caching(struct anv_device *device, uint32_t gem_handle, uint32_t caching);
1421 int anv_gem_set_domain(struct anv_device *device, uint32_t gem_handle,
1422 uint32_t read_domains, uint32_t write_domain);
1423 int anv_gem_sync_file_merge(struct anv_device *device, int fd1, int fd2);
1424 uint32_t anv_gem_syncobj_create(struct anv_device *device, uint32_t flags);
1425 void anv_gem_syncobj_destroy(struct anv_device *device, uint32_t handle);
1426 int anv_gem_syncobj_handle_to_fd(struct anv_device *device, uint32_t handle);
1427 uint32_t anv_gem_syncobj_fd_to_handle(struct anv_device *device, int fd);
1428 int anv_gem_syncobj_export_sync_file(struct anv_device *device,
1429 uint32_t handle);
1430 int anv_gem_syncobj_import_sync_file(struct anv_device *device,
1431 uint32_t handle, int fd);
1432 void anv_gem_syncobj_reset(struct anv_device *device, uint32_t handle);
1433 bool anv_gem_supports_syncobj_wait(int fd);
1434 int anv_gem_syncobj_wait(struct anv_device *device,
1435 uint32_t *handles, uint32_t num_handles,
1436 int64_t abs_timeout_ns, bool wait_all);
1437
1438 bool anv_vma_alloc(struct anv_device *device, struct anv_bo *bo);
1439 void anv_vma_free(struct anv_device *device, struct anv_bo *bo);
1440
1441 struct anv_reloc_list {
1442 uint32_t num_relocs;
1443 uint32_t array_length;
1444 struct drm_i915_gem_relocation_entry * relocs;
1445 struct anv_bo ** reloc_bos;
1446 uint32_t dep_words;
1447 BITSET_WORD * deps;
1448 };
1449
1450 VkResult anv_reloc_list_init(struct anv_reloc_list *list,
1451 const VkAllocationCallbacks *alloc);
1452 void anv_reloc_list_finish(struct anv_reloc_list *list,
1453 const VkAllocationCallbacks *alloc);
1454
1455 VkResult anv_reloc_list_add(struct anv_reloc_list *list,
1456 const VkAllocationCallbacks *alloc,
1457 uint32_t offset, struct anv_bo *target_bo,
1458 uint32_t delta, uint64_t *address_u64_out);
1459
1460 struct anv_batch_bo {
1461 /* Link in the anv_cmd_buffer.owned_batch_bos list */
1462 struct list_head link;
1463
1464 struct anv_bo * bo;
1465
1466 /* Bytes actually consumed in this batch BO */
1467 uint32_t length;
1468
1469 struct anv_reloc_list relocs;
1470 };
1471
1472 struct anv_batch {
1473 const VkAllocationCallbacks * alloc;
1474
1475 void * start;
1476 void * end;
1477 void * next;
1478
1479 struct anv_reloc_list * relocs;
1480
1481 /* This callback is called (with the associated user data) in the event
1482 * that the batch runs out of space.
1483 */
1484 VkResult (*extend_cb)(struct anv_batch *, void *);
1485 void * user_data;
1486
1487 /**
1488 * Current error status of the command buffer. Used to track inconsistent
1489 * or incomplete command buffer states that are the consequence of run-time
1490 * errors such as out of memory scenarios. We want to track this in the
1491 * batch because the command buffer object is not visible to some parts
1492 * of the driver.
1493 */
1494 VkResult status;
1495 };
1496
1497 void *anv_batch_emit_dwords(struct anv_batch *batch, int num_dwords);
1498 void anv_batch_emit_batch(struct anv_batch *batch, struct anv_batch *other);
1499 uint64_t anv_batch_emit_reloc(struct anv_batch *batch,
1500 void *location, struct anv_bo *bo, uint32_t offset);
1501
1502 static inline VkResult
1503 anv_batch_set_error(struct anv_batch *batch, VkResult error)
1504 {
1505 assert(error != VK_SUCCESS);
1506 if (batch->status == VK_SUCCESS)
1507 batch->status = error;
1508 return batch->status;
1509 }
1510
1511 static inline bool
1512 anv_batch_has_error(struct anv_batch *batch)
1513 {
1514 return batch->status != VK_SUCCESS;
1515 }
1516
1517 struct anv_address {
1518 struct anv_bo *bo;
1519 uint32_t offset;
1520 };
1521
1522 #define ANV_NULL_ADDRESS ((struct anv_address) { NULL, 0 })
1523
1524 static inline bool
1525 anv_address_is_null(struct anv_address addr)
1526 {
1527 return addr.bo == NULL && addr.offset == 0;
1528 }
1529
1530 static inline uint64_t
1531 anv_address_physical(struct anv_address addr)
1532 {
1533 if (addr.bo && (addr.bo->flags & EXEC_OBJECT_PINNED))
1534 return gen_canonical_address(addr.bo->offset + addr.offset);
1535 else
1536 return gen_canonical_address(addr.offset);
1537 }
1538
1539 static inline struct anv_address
1540 anv_address_add(struct anv_address addr, uint64_t offset)
1541 {
1542 addr.offset += offset;
1543 return addr;
1544 }
1545
1546 static inline void
1547 write_reloc(const struct anv_device *device, void *p, uint64_t v, bool flush)
1548 {
1549 unsigned reloc_size = 0;
1550 if (device->info.gen >= 8) {
1551 reloc_size = sizeof(uint64_t);
1552 *(uint64_t *)p = gen_canonical_address(v);
1553 } else {
1554 reloc_size = sizeof(uint32_t);
1555 *(uint32_t *)p = v;
1556 }
1557
1558 if (flush && !device->info.has_llc)
1559 gen_flush_range(p, reloc_size);
1560 }
1561
1562 static inline uint64_t
1563 _anv_combine_address(struct anv_batch *batch, void *location,
1564 const struct anv_address address, uint32_t delta)
1565 {
1566 if (address.bo == NULL) {
1567 return address.offset + delta;
1568 } else {
1569 assert(batch->start <= location && location < batch->end);
1570
1571 return anv_batch_emit_reloc(batch, location, address.bo, address.offset + delta);
1572 }
1573 }
1574
1575 #define __gen_address_type struct anv_address
1576 #define __gen_user_data struct anv_batch
1577 #define __gen_combine_address _anv_combine_address
1578
1579 /* Wrapper macros needed to work around preprocessor argument issues. In
1580 * particular, arguments don't get pre-evaluated if they are concatenated.
1581 * This means that, if you pass GENX(3DSTATE_PS) into the emit macro, the
1582 * GENX macro won't get evaluated if the emit macro contains "cmd ## foo".
1583 * We can work around this easily enough with these helpers.
1584 */
1585 #define __anv_cmd_length(cmd) cmd ## _length
1586 #define __anv_cmd_length_bias(cmd) cmd ## _length_bias
1587 #define __anv_cmd_header(cmd) cmd ## _header
1588 #define __anv_cmd_pack(cmd) cmd ## _pack
1589 #define __anv_reg_num(reg) reg ## _num
1590
1591 #define anv_pack_struct(dst, struc, ...) do { \
1592 struct struc __template = { \
1593 __VA_ARGS__ \
1594 }; \
1595 __anv_cmd_pack(struc)(NULL, dst, &__template); \
1596 VG(VALGRIND_CHECK_MEM_IS_DEFINED(dst, __anv_cmd_length(struc) * 4)); \
1597 } while (0)
1598
1599 #define anv_batch_emitn(batch, n, cmd, ...) ({ \
1600 void *__dst = anv_batch_emit_dwords(batch, n); \
1601 if (__dst) { \
1602 struct cmd __template = { \
1603 __anv_cmd_header(cmd), \
1604 .DWordLength = n - __anv_cmd_length_bias(cmd), \
1605 __VA_ARGS__ \
1606 }; \
1607 __anv_cmd_pack(cmd)(batch, __dst, &__template); \
1608 } \
1609 __dst; \
1610 })
1611
1612 #define anv_batch_emit_merge(batch, dwords0, dwords1) \
1613 do { \
1614 uint32_t *dw; \
1615 \
1616 STATIC_ASSERT(ARRAY_SIZE(dwords0) == ARRAY_SIZE(dwords1)); \
1617 dw = anv_batch_emit_dwords((batch), ARRAY_SIZE(dwords0)); \
1618 if (!dw) \
1619 break; \
1620 for (uint32_t i = 0; i < ARRAY_SIZE(dwords0); i++) \
1621 dw[i] = (dwords0)[i] | (dwords1)[i]; \
1622 VG(VALGRIND_CHECK_MEM_IS_DEFINED(dw, ARRAY_SIZE(dwords0) * 4));\
1623 } while (0)
1624
1625 #define anv_batch_emit(batch, cmd, name) \
1626 for (struct cmd name = { __anv_cmd_header(cmd) }, \
1627 *_dst = anv_batch_emit_dwords(batch, __anv_cmd_length(cmd)); \
1628 __builtin_expect(_dst != NULL, 1); \
1629 ({ __anv_cmd_pack(cmd)(batch, _dst, &name); \
1630 VG(VALGRIND_CHECK_MEM_IS_DEFINED(_dst, __anv_cmd_length(cmd) * 4)); \
1631 _dst = NULL; \
1632 }))
1633
1634 struct anv_device_memory {
1635 struct list_head link;
1636
1637 struct anv_bo * bo;
1638 struct anv_memory_type * type;
1639 VkDeviceSize map_size;
1640 void * map;
1641
1642 /* If set, we are holding reference to AHardwareBuffer
1643 * which we must release when memory is freed.
1644 */
1645 struct AHardwareBuffer * ahw;
1646
1647 /* If set, this memory comes from a host pointer. */
1648 void * host_ptr;
1649 };
1650
1651 /**
1652 * Header for Vertex URB Entry (VUE)
1653 */
1654 struct anv_vue_header {
1655 uint32_t Reserved;
1656 uint32_t RTAIndex; /* RenderTargetArrayIndex */
1657 uint32_t ViewportIndex;
1658 float PointWidth;
1659 };
1660
1661 /** Struct representing a sampled image descriptor
1662 *
1663 * This descriptor layout is used for sampled images, bare sampler, and
1664 * combined image/sampler descriptors.
1665 */
1666 struct anv_sampled_image_descriptor {
1667 /** Bindless image handle
1668 *
1669 * This is expected to already be shifted such that the 20-bit
1670 * SURFACE_STATE table index is in the top 20 bits.
1671 */
1672 uint32_t image;
1673
1674 /** Bindless sampler handle
1675 *
1676 * This is assumed to be a 32B-aligned SAMPLER_STATE pointer relative
1677 * to the dynamic state base address.
1678 */
1679 uint32_t sampler;
1680 };
1681
1682 struct anv_texture_swizzle_descriptor {
1683 /** Texture swizzle
1684 *
1685 * See also nir_intrinsic_channel_select_intel
1686 */
1687 uint8_t swizzle[4];
1688
1689 /** Unused padding to ensure the struct is a multiple of 64 bits */
1690 uint32_t _pad;
1691 };
1692
1693 /** Struct representing a storage image descriptor */
1694 struct anv_storage_image_descriptor {
1695 /** Bindless image handles
1696 *
1697 * These are expected to already be shifted such that the 20-bit
1698 * SURFACE_STATE table index is in the top 20 bits.
1699 */
1700 uint32_t read_write;
1701 uint32_t write_only;
1702 };
1703
1704 /** Struct representing a address/range descriptor
1705 *
1706 * The fields of this struct correspond directly to the data layout of
1707 * nir_address_format_64bit_bounded_global addresses. The last field is the
1708 * offset in the NIR address so it must be zero so that when you load the
1709 * descriptor you get a pointer to the start of the range.
1710 */
1711 struct anv_address_range_descriptor {
1712 uint64_t address;
1713 uint32_t range;
1714 uint32_t zero;
1715 };
1716
1717 enum anv_descriptor_data {
1718 /** The descriptor contains a BTI reference to a surface state */
1719 ANV_DESCRIPTOR_SURFACE_STATE = (1 << 0),
1720 /** The descriptor contains a BTI reference to a sampler state */
1721 ANV_DESCRIPTOR_SAMPLER_STATE = (1 << 1),
1722 /** The descriptor contains an actual buffer view */
1723 ANV_DESCRIPTOR_BUFFER_VIEW = (1 << 2),
1724 /** The descriptor contains auxiliary image layout data */
1725 ANV_DESCRIPTOR_IMAGE_PARAM = (1 << 3),
1726 /** The descriptor contains auxiliary image layout data */
1727 ANV_DESCRIPTOR_INLINE_UNIFORM = (1 << 4),
1728 /** anv_address_range_descriptor with a buffer address and range */
1729 ANV_DESCRIPTOR_ADDRESS_RANGE = (1 << 5),
1730 /** Bindless surface handle */
1731 ANV_DESCRIPTOR_SAMPLED_IMAGE = (1 << 6),
1732 /** Storage image handles */
1733 ANV_DESCRIPTOR_STORAGE_IMAGE = (1 << 7),
1734 /** Storage image handles */
1735 ANV_DESCRIPTOR_TEXTURE_SWIZZLE = (1 << 8),
1736 };
1737
1738 struct anv_descriptor_set_binding_layout {
1739 #ifndef NDEBUG
1740 /* The type of the descriptors in this binding */
1741 VkDescriptorType type;
1742 #endif
1743
1744 /* Flags provided when this binding was created */
1745 VkDescriptorBindingFlagsEXT flags;
1746
1747 /* Bitfield representing the type of data this descriptor contains */
1748 enum anv_descriptor_data data;
1749
1750 /* Maximum number of YCbCr texture/sampler planes */
1751 uint8_t max_plane_count;
1752
1753 /* Number of array elements in this binding (or size in bytes for inline
1754 * uniform data)
1755 */
1756 uint16_t array_size;
1757
1758 /* Index into the flattend descriptor set */
1759 uint16_t descriptor_index;
1760
1761 /* Index into the dynamic state array for a dynamic buffer */
1762 int16_t dynamic_offset_index;
1763
1764 /* Index into the descriptor set buffer views */
1765 int16_t buffer_view_index;
1766
1767 /* Offset into the descriptor buffer where this descriptor lives */
1768 uint32_t descriptor_offset;
1769
1770 /* Immutable samplers (or NULL if no immutable samplers) */
1771 struct anv_sampler **immutable_samplers;
1772 };
1773
1774 unsigned anv_descriptor_size(const struct anv_descriptor_set_binding_layout *layout);
1775
1776 unsigned anv_descriptor_type_size(const struct anv_physical_device *pdevice,
1777 VkDescriptorType type);
1778
1779 bool anv_descriptor_supports_bindless(const struct anv_physical_device *pdevice,
1780 const struct anv_descriptor_set_binding_layout *binding,
1781 bool sampler);
1782
1783 bool anv_descriptor_requires_bindless(const struct anv_physical_device *pdevice,
1784 const struct anv_descriptor_set_binding_layout *binding,
1785 bool sampler);
1786
1787 struct anv_descriptor_set_layout {
1788 /* Descriptor set layouts can be destroyed at almost any time */
1789 uint32_t ref_cnt;
1790
1791 /* Number of bindings in this descriptor set */
1792 uint16_t binding_count;
1793
1794 /* Total size of the descriptor set with room for all array entries */
1795 uint16_t size;
1796
1797 /* Shader stages affected by this descriptor set */
1798 uint16_t shader_stages;
1799
1800 /* Number of buffer views in this descriptor set */
1801 uint16_t buffer_view_count;
1802
1803 /* Number of dynamic offsets used by this descriptor set */
1804 uint16_t dynamic_offset_count;
1805
1806 /* Size of the descriptor buffer for this descriptor set */
1807 uint32_t descriptor_buffer_size;
1808
1809 /* Bindings in this descriptor set */
1810 struct anv_descriptor_set_binding_layout binding[0];
1811 };
1812
1813 static inline void
1814 anv_descriptor_set_layout_ref(struct anv_descriptor_set_layout *layout)
1815 {
1816 assert(layout && layout->ref_cnt >= 1);
1817 p_atomic_inc(&layout->ref_cnt);
1818 }
1819
1820 static inline void
1821 anv_descriptor_set_layout_unref(struct anv_device *device,
1822 struct anv_descriptor_set_layout *layout)
1823 {
1824 assert(layout && layout->ref_cnt >= 1);
1825 if (p_atomic_dec_zero(&layout->ref_cnt))
1826 vk_free(&device->alloc, layout);
1827 }
1828
1829 struct anv_descriptor {
1830 VkDescriptorType type;
1831
1832 union {
1833 struct {
1834 VkImageLayout layout;
1835 struct anv_image_view *image_view;
1836 struct anv_sampler *sampler;
1837 };
1838
1839 struct {
1840 struct anv_buffer *buffer;
1841 uint64_t offset;
1842 uint64_t range;
1843 };
1844
1845 struct anv_buffer_view *buffer_view;
1846 };
1847 };
1848
1849 struct anv_descriptor_set {
1850 struct anv_descriptor_pool *pool;
1851 struct anv_descriptor_set_layout *layout;
1852 uint32_t size;
1853
1854 /* State relative to anv_descriptor_pool::bo */
1855 struct anv_state desc_mem;
1856 /* Surface state for the descriptor buffer */
1857 struct anv_state desc_surface_state;
1858
1859 uint32_t buffer_view_count;
1860 struct anv_buffer_view *buffer_views;
1861
1862 /* Link to descriptor pool's desc_sets list . */
1863 struct list_head pool_link;
1864
1865 struct anv_descriptor descriptors[0];
1866 };
1867
1868 struct anv_buffer_view {
1869 enum isl_format format; /**< VkBufferViewCreateInfo::format */
1870 uint64_t range; /**< VkBufferViewCreateInfo::range */
1871
1872 struct anv_address address;
1873
1874 struct anv_state surface_state;
1875 struct anv_state storage_surface_state;
1876 struct anv_state writeonly_storage_surface_state;
1877
1878 struct brw_image_param storage_image_param;
1879 };
1880
1881 struct anv_push_descriptor_set {
1882 struct anv_descriptor_set set;
1883
1884 /* Put this field right behind anv_descriptor_set so it fills up the
1885 * descriptors[0] field. */
1886 struct anv_descriptor descriptors[MAX_PUSH_DESCRIPTORS];
1887
1888 /** True if the descriptor set buffer has been referenced by a draw or
1889 * dispatch command.
1890 */
1891 bool set_used_on_gpu;
1892
1893 struct anv_buffer_view buffer_views[MAX_PUSH_DESCRIPTORS];
1894 };
1895
1896 struct anv_descriptor_pool {
1897 uint32_t size;
1898 uint32_t next;
1899 uint32_t free_list;
1900
1901 struct anv_bo *bo;
1902 struct util_vma_heap bo_heap;
1903
1904 struct anv_state_stream surface_state_stream;
1905 void *surface_state_free_list;
1906
1907 struct list_head desc_sets;
1908
1909 char data[0];
1910 };
1911
1912 enum anv_descriptor_template_entry_type {
1913 ANV_DESCRIPTOR_TEMPLATE_ENTRY_TYPE_IMAGE,
1914 ANV_DESCRIPTOR_TEMPLATE_ENTRY_TYPE_BUFFER,
1915 ANV_DESCRIPTOR_TEMPLATE_ENTRY_TYPE_BUFFER_VIEW
1916 };
1917
1918 struct anv_descriptor_template_entry {
1919 /* The type of descriptor in this entry */
1920 VkDescriptorType type;
1921
1922 /* Binding in the descriptor set */
1923 uint32_t binding;
1924
1925 /* Offset at which to write into the descriptor set binding */
1926 uint32_t array_element;
1927
1928 /* Number of elements to write into the descriptor set binding */
1929 uint32_t array_count;
1930
1931 /* Offset into the user provided data */
1932 size_t offset;
1933
1934 /* Stride between elements into the user provided data */
1935 size_t stride;
1936 };
1937
1938 struct anv_descriptor_update_template {
1939 VkPipelineBindPoint bind_point;
1940
1941 /* The descriptor set this template corresponds to. This value is only
1942 * valid if the template was created with the templateType
1943 * VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET.
1944 */
1945 uint8_t set;
1946
1947 /* Number of entries in this template */
1948 uint32_t entry_count;
1949
1950 /* Entries of the template */
1951 struct anv_descriptor_template_entry entries[0];
1952 };
1953
1954 size_t
1955 anv_descriptor_set_layout_size(const struct anv_descriptor_set_layout *layout);
1956
1957 void
1958 anv_descriptor_set_write_image_view(struct anv_device *device,
1959 struct anv_descriptor_set *set,
1960 const VkDescriptorImageInfo * const info,
1961 VkDescriptorType type,
1962 uint32_t binding,
1963 uint32_t element);
1964
1965 void
1966 anv_descriptor_set_write_buffer_view(struct anv_device *device,
1967 struct anv_descriptor_set *set,
1968 VkDescriptorType type,
1969 struct anv_buffer_view *buffer_view,
1970 uint32_t binding,
1971 uint32_t element);
1972
1973 void
1974 anv_descriptor_set_write_buffer(struct anv_device *device,
1975 struct anv_descriptor_set *set,
1976 struct anv_state_stream *alloc_stream,
1977 VkDescriptorType type,
1978 struct anv_buffer *buffer,
1979 uint32_t binding,
1980 uint32_t element,
1981 VkDeviceSize offset,
1982 VkDeviceSize range);
1983 void
1984 anv_descriptor_set_write_inline_uniform_data(struct anv_device *device,
1985 struct anv_descriptor_set *set,
1986 uint32_t binding,
1987 const void *data,
1988 size_t offset,
1989 size_t size);
1990
1991 void
1992 anv_descriptor_set_write_template(struct anv_device *device,
1993 struct anv_descriptor_set *set,
1994 struct anv_state_stream *alloc_stream,
1995 const struct anv_descriptor_update_template *template,
1996 const void *data);
1997
1998 VkResult
1999 anv_descriptor_set_create(struct anv_device *device,
2000 struct anv_descriptor_pool *pool,
2001 struct anv_descriptor_set_layout *layout,
2002 struct anv_descriptor_set **out_set);
2003
2004 void
2005 anv_descriptor_set_destroy(struct anv_device *device,
2006 struct anv_descriptor_pool *pool,
2007 struct anv_descriptor_set *set);
2008
2009 #define ANV_DESCRIPTOR_SET_DESCRIPTORS (UINT8_MAX - 3)
2010 #define ANV_DESCRIPTOR_SET_NUM_WORK_GROUPS (UINT8_MAX - 2)
2011 #define ANV_DESCRIPTOR_SET_SHADER_CONSTANTS (UINT8_MAX - 1)
2012 #define ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS UINT8_MAX
2013
2014 struct anv_pipeline_binding {
2015 /** Index in the descriptor set
2016 *
2017 * This is a flattened index; the descriptor set layout is already taken
2018 * into account.
2019 */
2020 uint32_t index;
2021
2022 /** The descriptor set this surface corresponds to.
2023 *
2024 * The special ANV_DESCRIPTOR_SET_* values above indicates that this
2025 * binding is not a normal descriptor set but something else.
2026 */
2027 uint8_t set;
2028
2029 union {
2030 /** Plane in the binding index for images */
2031 uint8_t plane;
2032
2033 /** Input attachment index (relative to the subpass) */
2034 uint8_t input_attachment_index;
2035
2036 /** Dynamic offset index (for dynamic UBOs and SSBOs) */
2037 uint8_t dynamic_offset_index;
2038 };
2039
2040 /** For a storage image, whether it is write-only */
2041 bool write_only;
2042 };
2043
2044 struct anv_pipeline_layout {
2045 struct {
2046 struct anv_descriptor_set_layout *layout;
2047 uint32_t dynamic_offset_start;
2048 } set[MAX_SETS];
2049
2050 uint32_t num_sets;
2051
2052 unsigned char sha1[20];
2053 };
2054
2055 struct anv_buffer {
2056 struct anv_device * device;
2057 VkDeviceSize size;
2058
2059 VkBufferUsageFlags usage;
2060
2061 /* Set when bound */
2062 struct anv_address address;
2063 };
2064
2065 static inline uint64_t
2066 anv_buffer_get_range(struct anv_buffer *buffer, uint64_t offset, uint64_t range)
2067 {
2068 assert(offset <= buffer->size);
2069 if (range == VK_WHOLE_SIZE) {
2070 return buffer->size - offset;
2071 } else {
2072 assert(range + offset >= range);
2073 assert(range + offset <= buffer->size);
2074 return range;
2075 }
2076 }
2077
2078 enum anv_cmd_dirty_bits {
2079 ANV_CMD_DIRTY_DYNAMIC_VIEWPORT = 1 << 0, /* VK_DYNAMIC_STATE_VIEWPORT */
2080 ANV_CMD_DIRTY_DYNAMIC_SCISSOR = 1 << 1, /* VK_DYNAMIC_STATE_SCISSOR */
2081 ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH = 1 << 2, /* VK_DYNAMIC_STATE_LINE_WIDTH */
2082 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS = 1 << 3, /* VK_DYNAMIC_STATE_DEPTH_BIAS */
2083 ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS = 1 << 4, /* VK_DYNAMIC_STATE_BLEND_CONSTANTS */
2084 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS = 1 << 5, /* VK_DYNAMIC_STATE_DEPTH_BOUNDS */
2085 ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK = 1 << 6, /* VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK */
2086 ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK = 1 << 7, /* VK_DYNAMIC_STATE_STENCIL_WRITE_MASK */
2087 ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE = 1 << 8, /* VK_DYNAMIC_STATE_STENCIL_REFERENCE */
2088 ANV_CMD_DIRTY_PIPELINE = 1 << 9,
2089 ANV_CMD_DIRTY_INDEX_BUFFER = 1 << 10,
2090 ANV_CMD_DIRTY_RENDER_TARGETS = 1 << 11,
2091 ANV_CMD_DIRTY_XFB_ENABLE = 1 << 12,
2092 ANV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE = 1 << 13, /* VK_DYNAMIC_STATE_LINE_STIPPLE_EXT */
2093 };
2094 typedef uint32_t anv_cmd_dirty_mask_t;
2095
2096 #define ANV_CMD_DIRTY_DYNAMIC_ALL \
2097 (ANV_CMD_DIRTY_DYNAMIC_VIEWPORT | \
2098 ANV_CMD_DIRTY_DYNAMIC_SCISSOR | \
2099 ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH | \
2100 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS | \
2101 ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS | \
2102 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS | \
2103 ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK | \
2104 ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK | \
2105 ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE | \
2106 ANV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE)
2107
2108 static inline enum anv_cmd_dirty_bits
2109 anv_cmd_dirty_bit_for_vk_dynamic_state(VkDynamicState vk_state)
2110 {
2111 switch (vk_state) {
2112 case VK_DYNAMIC_STATE_VIEWPORT:
2113 return ANV_CMD_DIRTY_DYNAMIC_VIEWPORT;
2114 case VK_DYNAMIC_STATE_SCISSOR:
2115 return ANV_CMD_DIRTY_DYNAMIC_SCISSOR;
2116 case VK_DYNAMIC_STATE_LINE_WIDTH:
2117 return ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH;
2118 case VK_DYNAMIC_STATE_DEPTH_BIAS:
2119 return ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS;
2120 case VK_DYNAMIC_STATE_BLEND_CONSTANTS:
2121 return ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS;
2122 case VK_DYNAMIC_STATE_DEPTH_BOUNDS:
2123 return ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS;
2124 case VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK:
2125 return ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK;
2126 case VK_DYNAMIC_STATE_STENCIL_WRITE_MASK:
2127 return ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK;
2128 case VK_DYNAMIC_STATE_STENCIL_REFERENCE:
2129 return ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE;
2130 case VK_DYNAMIC_STATE_LINE_STIPPLE_EXT:
2131 return ANV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE;
2132 default:
2133 assert(!"Unsupported dynamic state");
2134 return 0;
2135 }
2136 }
2137
2138
2139 enum anv_pipe_bits {
2140 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT = (1 << 0),
2141 ANV_PIPE_STALL_AT_SCOREBOARD_BIT = (1 << 1),
2142 ANV_PIPE_STATE_CACHE_INVALIDATE_BIT = (1 << 2),
2143 ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT = (1 << 3),
2144 ANV_PIPE_VF_CACHE_INVALIDATE_BIT = (1 << 4),
2145 ANV_PIPE_DATA_CACHE_FLUSH_BIT = (1 << 5),
2146 ANV_PIPE_TILE_CACHE_FLUSH_BIT = (1 << 6),
2147 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT = (1 << 10),
2148 ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT = (1 << 11),
2149 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT = (1 << 12),
2150 ANV_PIPE_DEPTH_STALL_BIT = (1 << 13),
2151 ANV_PIPE_CS_STALL_BIT = (1 << 20),
2152
2153 /* This bit does not exist directly in PIPE_CONTROL. Instead it means that
2154 * a flush has happened but not a CS stall. The next time we do any sort
2155 * of invalidation we need to insert a CS stall at that time. Otherwise,
2156 * we would have to CS stall on every flush which could be bad.
2157 */
2158 ANV_PIPE_NEEDS_CS_STALL_BIT = (1 << 21),
2159
2160 /* This bit does not exist directly in PIPE_CONTROL. It means that render
2161 * target operations related to transfer commands with VkBuffer as
2162 * destination are ongoing. Some operations like copies on the command
2163 * streamer might need to be aware of this to trigger the appropriate stall
2164 * before they can proceed with the copy.
2165 */
2166 ANV_PIPE_RENDER_TARGET_BUFFER_WRITES = (1 << 22),
2167 };
2168
2169 #define ANV_PIPE_FLUSH_BITS ( \
2170 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | \
2171 ANV_PIPE_DATA_CACHE_FLUSH_BIT | \
2172 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | \
2173 ANV_PIPE_TILE_CACHE_FLUSH_BIT)
2174
2175 #define ANV_PIPE_STALL_BITS ( \
2176 ANV_PIPE_STALL_AT_SCOREBOARD_BIT | \
2177 ANV_PIPE_DEPTH_STALL_BIT | \
2178 ANV_PIPE_CS_STALL_BIT)
2179
2180 #define ANV_PIPE_INVALIDATE_BITS ( \
2181 ANV_PIPE_STATE_CACHE_INVALIDATE_BIT | \
2182 ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT | \
2183 ANV_PIPE_VF_CACHE_INVALIDATE_BIT | \
2184 ANV_PIPE_DATA_CACHE_FLUSH_BIT | \
2185 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT | \
2186 ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT)
2187
2188 static inline enum anv_pipe_bits
2189 anv_pipe_flush_bits_for_access_flags(VkAccessFlags flags)
2190 {
2191 enum anv_pipe_bits pipe_bits = 0;
2192
2193 unsigned b;
2194 for_each_bit(b, flags) {
2195 switch ((VkAccessFlagBits)(1 << b)) {
2196 case VK_ACCESS_SHADER_WRITE_BIT:
2197 /* We're transitioning a buffer that was previously used as write
2198 * destination through the data port. To make its content available
2199 * to future operations, flush the data cache.
2200 */
2201 pipe_bits |= ANV_PIPE_DATA_CACHE_FLUSH_BIT;
2202 break;
2203 case VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT:
2204 /* We're transitioning a buffer that was previously used as render
2205 * target. To make its content available to future operations, flush
2206 * the render target cache.
2207 */
2208 pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
2209 break;
2210 case VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT:
2211 /* We're transitioning a buffer that was previously used as depth
2212 * buffer. To make its content available to future operations, flush
2213 * the depth cache.
2214 */
2215 pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
2216 break;
2217 case VK_ACCESS_TRANSFER_WRITE_BIT:
2218 /* We're transitioning a buffer that was previously used as a
2219 * transfer write destination. Generic write operations include color
2220 * & depth operations as well as buffer operations like :
2221 * - vkCmdClearColorImage()
2222 * - vkCmdClearDepthStencilImage()
2223 * - vkCmdBlitImage()
2224 * - vkCmdCopy*(), vkCmdUpdate*(), vkCmdFill*()
2225 *
2226 * Most of these operations are implemented using Blorp which writes
2227 * through the render target, so flush that cache to make it visible
2228 * to future operations. And for depth related operations we also
2229 * need to flush the depth cache.
2230 */
2231 pipe_bits |= ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
2232 pipe_bits |= ANV_PIPE_DEPTH_CACHE_FLUSH_BIT;
2233 break;
2234 case VK_ACCESS_MEMORY_WRITE_BIT:
2235 /* We're transitioning a buffer for generic write operations. Flush
2236 * all the caches.
2237 */
2238 pipe_bits |= ANV_PIPE_FLUSH_BITS;
2239 break;
2240 default:
2241 break; /* Nothing to do */
2242 }
2243 }
2244
2245 return pipe_bits;
2246 }
2247
2248 static inline enum anv_pipe_bits
2249 anv_pipe_invalidate_bits_for_access_flags(VkAccessFlags flags)
2250 {
2251 enum anv_pipe_bits pipe_bits = 0;
2252
2253 unsigned b;
2254 for_each_bit(b, flags) {
2255 switch ((VkAccessFlagBits)(1 << b)) {
2256 case VK_ACCESS_INDIRECT_COMMAND_READ_BIT:
2257 /* Indirect draw commands take a buffer as input that we're going to
2258 * read from the command streamer to load some of the HW registers
2259 * (see genX_cmd_buffer.c:load_indirect_parameters). This requires a
2260 * command streamer stall so that all the cache flushes have
2261 * completed before the command streamer loads from memory.
2262 */
2263 pipe_bits |= ANV_PIPE_CS_STALL_BIT;
2264 /* Indirect draw commands also set gl_BaseVertex & gl_BaseIndex
2265 * through a vertex buffer, so invalidate that cache.
2266 */
2267 pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
2268 /* For CmdDipatchIndirect, we also load gl_NumWorkGroups through a
2269 * UBO from the buffer, so we need to invalidate constant cache.
2270 */
2271 pipe_bits |= ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
2272 break;
2273 case VK_ACCESS_INDEX_READ_BIT:
2274 case VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT:
2275 /* We transitioning a buffer to be used for as input for vkCmdDraw*
2276 * commands, so we invalidate the VF cache to make sure there is no
2277 * stale data when we start rendering.
2278 */
2279 pipe_bits |= ANV_PIPE_VF_CACHE_INVALIDATE_BIT;
2280 break;
2281 case VK_ACCESS_UNIFORM_READ_BIT:
2282 /* We transitioning a buffer to be used as uniform data. Because
2283 * uniform is accessed through the data port & sampler, we need to
2284 * invalidate the texture cache (sampler) & constant cache (data
2285 * port) to avoid stale data.
2286 */
2287 pipe_bits |= ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT;
2288 pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
2289 break;
2290 case VK_ACCESS_SHADER_READ_BIT:
2291 case VK_ACCESS_INPUT_ATTACHMENT_READ_BIT:
2292 case VK_ACCESS_TRANSFER_READ_BIT:
2293 /* Transitioning a buffer to be read through the sampler, so
2294 * invalidate the texture cache, we don't want any stale data.
2295 */
2296 pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT;
2297 break;
2298 case VK_ACCESS_MEMORY_READ_BIT:
2299 /* Transitioning a buffer for generic read, invalidate all the
2300 * caches.
2301 */
2302 pipe_bits |= ANV_PIPE_INVALIDATE_BITS;
2303 break;
2304 case VK_ACCESS_MEMORY_WRITE_BIT:
2305 /* Generic write, make sure all previously written things land in
2306 * memory.
2307 */
2308 pipe_bits |= ANV_PIPE_FLUSH_BITS;
2309 break;
2310 case VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT:
2311 /* Transitioning a buffer for conditional rendering. We'll load the
2312 * content of this buffer into HW registers using the command
2313 * streamer, so we need to stall the command streamer to make sure
2314 * any in-flight flush operations have completed.
2315 */
2316 pipe_bits |= ANV_PIPE_CS_STALL_BIT;
2317 break;
2318 default:
2319 break; /* Nothing to do */
2320 }
2321 }
2322
2323 return pipe_bits;
2324 }
2325
2326 #define VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV ( \
2327 VK_IMAGE_ASPECT_COLOR_BIT | \
2328 VK_IMAGE_ASPECT_PLANE_0_BIT | \
2329 VK_IMAGE_ASPECT_PLANE_1_BIT | \
2330 VK_IMAGE_ASPECT_PLANE_2_BIT)
2331 #define VK_IMAGE_ASPECT_PLANES_BITS_ANV ( \
2332 VK_IMAGE_ASPECT_PLANE_0_BIT | \
2333 VK_IMAGE_ASPECT_PLANE_1_BIT | \
2334 VK_IMAGE_ASPECT_PLANE_2_BIT)
2335
2336 struct anv_vertex_binding {
2337 struct anv_buffer * buffer;
2338 VkDeviceSize offset;
2339 };
2340
2341 struct anv_xfb_binding {
2342 struct anv_buffer * buffer;
2343 VkDeviceSize offset;
2344 VkDeviceSize size;
2345 };
2346
2347 #define ANV_PARAM_PUSH(offset) ((1 << 16) | (uint32_t)(offset))
2348 #define ANV_PARAM_IS_PUSH(param) ((uint32_t)(param) >> 16 == 1)
2349 #define ANV_PARAM_PUSH_OFFSET(param) ((param) & 0xffff)
2350
2351 #define ANV_PARAM_DYN_OFFSET(offset) ((2 << 16) | (uint32_t)(offset))
2352 #define ANV_PARAM_IS_DYN_OFFSET(param) ((uint32_t)(param) >> 16 == 2)
2353 #define ANV_PARAM_DYN_OFFSET_IDX(param) ((param) & 0xffff)
2354
2355 struct anv_push_constants {
2356 /* Push constant data provided by the client through vkPushConstants */
2357 uint8_t client_data[MAX_PUSH_CONSTANTS_SIZE];
2358
2359 /* Used for vkCmdDispatchBase */
2360 uint32_t base_work_group_id[3];
2361 };
2362
2363 struct anv_dynamic_state {
2364 struct {
2365 uint32_t count;
2366 VkViewport viewports[MAX_VIEWPORTS];
2367 } viewport;
2368
2369 struct {
2370 uint32_t count;
2371 VkRect2D scissors[MAX_SCISSORS];
2372 } scissor;
2373
2374 float line_width;
2375
2376 struct {
2377 float bias;
2378 float clamp;
2379 float slope;
2380 } depth_bias;
2381
2382 float blend_constants[4];
2383
2384 struct {
2385 float min;
2386 float max;
2387 } depth_bounds;
2388
2389 struct {
2390 uint32_t front;
2391 uint32_t back;
2392 } stencil_compare_mask;
2393
2394 struct {
2395 uint32_t front;
2396 uint32_t back;
2397 } stencil_write_mask;
2398
2399 struct {
2400 uint32_t front;
2401 uint32_t back;
2402 } stencil_reference;
2403
2404 struct {
2405 uint32_t factor;
2406 uint16_t pattern;
2407 } line_stipple;
2408 };
2409
2410 extern const struct anv_dynamic_state default_dynamic_state;
2411
2412 uint32_t anv_dynamic_state_copy(struct anv_dynamic_state *dest,
2413 const struct anv_dynamic_state *src,
2414 uint32_t copy_mask);
2415
2416 struct anv_surface_state {
2417 struct anv_state state;
2418 /** Address of the surface referred to by this state
2419 *
2420 * This address is relative to the start of the BO.
2421 */
2422 struct anv_address address;
2423 /* Address of the aux surface, if any
2424 *
2425 * This field is ANV_NULL_ADDRESS if and only if no aux surface exists.
2426 *
2427 * With the exception of gen8, the bottom 12 bits of this address' offset
2428 * include extra aux information.
2429 */
2430 struct anv_address aux_address;
2431 /* Address of the clear color, if any
2432 *
2433 * This address is relative to the start of the BO.
2434 */
2435 struct anv_address clear_address;
2436 };
2437
2438 /**
2439 * Attachment state when recording a renderpass instance.
2440 *
2441 * The clear value is valid only if there exists a pending clear.
2442 */
2443 struct anv_attachment_state {
2444 enum isl_aux_usage aux_usage;
2445 enum isl_aux_usage input_aux_usage;
2446 struct anv_surface_state color;
2447 struct anv_surface_state input;
2448
2449 VkImageLayout current_layout;
2450 VkImageLayout current_stencil_layout;
2451 VkImageAspectFlags pending_clear_aspects;
2452 VkImageAspectFlags pending_load_aspects;
2453 bool fast_clear;
2454 VkClearValue clear_value;
2455 bool clear_color_is_zero_one;
2456 bool clear_color_is_zero;
2457
2458 /* When multiview is active, attachments with a renderpass clear
2459 * operation have their respective layers cleared on the first
2460 * subpass that uses them, and only in that subpass. We keep track
2461 * of this using a bitfield to indicate which layers of an attachment
2462 * have not been cleared yet when multiview is active.
2463 */
2464 uint32_t pending_clear_views;
2465 struct anv_image_view * image_view;
2466 };
2467
2468 /** State tracking for particular pipeline bind point
2469 *
2470 * This struct is the base struct for anv_cmd_graphics_state and
2471 * anv_cmd_compute_state. These are used to track state which is bound to a
2472 * particular type of pipeline. Generic state that applies per-stage such as
2473 * binding table offsets and push constants is tracked generically with a
2474 * per-stage array in anv_cmd_state.
2475 */
2476 struct anv_cmd_pipeline_state {
2477 struct anv_pipeline *pipeline;
2478
2479 struct anv_descriptor_set *descriptors[MAX_SETS];
2480 uint32_t dynamic_offsets[MAX_DYNAMIC_BUFFERS];
2481
2482 struct anv_push_descriptor_set *push_descriptors[MAX_SETS];
2483 };
2484
2485 /** State tracking for graphics pipeline
2486 *
2487 * This has anv_cmd_pipeline_state as a base struct to track things which get
2488 * bound to a graphics pipeline. Along with general pipeline bind point state
2489 * which is in the anv_cmd_pipeline_state base struct, it also contains other
2490 * state which is graphics-specific.
2491 */
2492 struct anv_cmd_graphics_state {
2493 struct anv_cmd_pipeline_state base;
2494
2495 anv_cmd_dirty_mask_t dirty;
2496 uint32_t vb_dirty;
2497
2498 struct anv_dynamic_state dynamic;
2499
2500 struct {
2501 struct anv_buffer *index_buffer;
2502 uint32_t index_type; /**< 3DSTATE_INDEX_BUFFER.IndexFormat */
2503 uint32_t index_offset;
2504 } gen7;
2505 };
2506
2507 /** State tracking for compute pipeline
2508 *
2509 * This has anv_cmd_pipeline_state as a base struct to track things which get
2510 * bound to a compute pipeline. Along with general pipeline bind point state
2511 * which is in the anv_cmd_pipeline_state base struct, it also contains other
2512 * state which is compute-specific.
2513 */
2514 struct anv_cmd_compute_state {
2515 struct anv_cmd_pipeline_state base;
2516
2517 bool pipeline_dirty;
2518
2519 struct anv_address num_workgroups;
2520 };
2521
2522 /** State required while building cmd buffer */
2523 struct anv_cmd_state {
2524 /* PIPELINE_SELECT.PipelineSelection */
2525 uint32_t current_pipeline;
2526 const struct gen_l3_config * current_l3_config;
2527 uint32_t last_aux_map_state;
2528
2529 struct anv_cmd_graphics_state gfx;
2530 struct anv_cmd_compute_state compute;
2531
2532 enum anv_pipe_bits pending_pipe_bits;
2533 VkShaderStageFlags descriptors_dirty;
2534 VkShaderStageFlags push_constants_dirty;
2535
2536 struct anv_framebuffer * framebuffer;
2537 struct anv_render_pass * pass;
2538 struct anv_subpass * subpass;
2539 VkRect2D render_area;
2540 uint32_t restart_index;
2541 struct anv_vertex_binding vertex_bindings[MAX_VBS];
2542 bool xfb_enabled;
2543 struct anv_xfb_binding xfb_bindings[MAX_XFB_BUFFERS];
2544 VkShaderStageFlags push_constant_stages;
2545 struct anv_push_constants push_constants[MESA_SHADER_STAGES];
2546 struct anv_state binding_tables[MESA_SHADER_STAGES];
2547 struct anv_state samplers[MESA_SHADER_STAGES];
2548
2549 /**
2550 * Whether or not the gen8 PMA fix is enabled. We ensure that, at the top
2551 * of any command buffer it is disabled by disabling it in EndCommandBuffer
2552 * and before invoking the secondary in ExecuteCommands.
2553 */
2554 bool pma_fix_enabled;
2555
2556 /**
2557 * Whether or not we know for certain that HiZ is enabled for the current
2558 * subpass. If, for whatever reason, we are unsure as to whether HiZ is
2559 * enabled or not, this will be false.
2560 */
2561 bool hiz_enabled;
2562
2563 bool conditional_render_enabled;
2564
2565 /**
2566 * Last rendering scale argument provided to
2567 * genX(cmd_buffer_emit_hashing_mode)().
2568 */
2569 unsigned current_hash_scale;
2570
2571 /**
2572 * Array length is anv_cmd_state::pass::attachment_count. Array content is
2573 * valid only when recording a render pass instance.
2574 */
2575 struct anv_attachment_state * attachments;
2576
2577 /**
2578 * Surface states for color render targets. These are stored in a single
2579 * flat array. For depth-stencil attachments, the surface state is simply
2580 * left blank.
2581 */
2582 struct anv_state render_pass_states;
2583
2584 /**
2585 * A null surface state of the right size to match the framebuffer. This
2586 * is one of the states in render_pass_states.
2587 */
2588 struct anv_state null_surface_state;
2589 };
2590
2591 struct anv_cmd_pool {
2592 VkAllocationCallbacks alloc;
2593 struct list_head cmd_buffers;
2594 };
2595
2596 #define ANV_CMD_BUFFER_BATCH_SIZE 8192
2597
2598 enum anv_cmd_buffer_exec_mode {
2599 ANV_CMD_BUFFER_EXEC_MODE_PRIMARY,
2600 ANV_CMD_BUFFER_EXEC_MODE_EMIT,
2601 ANV_CMD_BUFFER_EXEC_MODE_GROW_AND_EMIT,
2602 ANV_CMD_BUFFER_EXEC_MODE_CHAIN,
2603 ANV_CMD_BUFFER_EXEC_MODE_COPY_AND_CHAIN,
2604 };
2605
2606 struct anv_cmd_buffer {
2607 VK_LOADER_DATA _loader_data;
2608
2609 struct anv_device * device;
2610
2611 struct anv_cmd_pool * pool;
2612 struct list_head pool_link;
2613
2614 struct anv_batch batch;
2615
2616 /* Fields required for the actual chain of anv_batch_bo's.
2617 *
2618 * These fields are initialized by anv_cmd_buffer_init_batch_bo_chain().
2619 */
2620 struct list_head batch_bos;
2621 enum anv_cmd_buffer_exec_mode exec_mode;
2622
2623 /* A vector of anv_batch_bo pointers for every batch or surface buffer
2624 * referenced by this command buffer
2625 *
2626 * initialized by anv_cmd_buffer_init_batch_bo_chain()
2627 */
2628 struct u_vector seen_bbos;
2629
2630 /* A vector of int32_t's for every block of binding tables.
2631 *
2632 * initialized by anv_cmd_buffer_init_batch_bo_chain()
2633 */
2634 struct u_vector bt_block_states;
2635 uint32_t bt_next;
2636
2637 struct anv_reloc_list surface_relocs;
2638 /** Last seen surface state block pool center bo offset */
2639 uint32_t last_ss_pool_center;
2640
2641 /* Serial for tracking buffer completion */
2642 uint32_t serial;
2643
2644 /* Stream objects for storing temporary data */
2645 struct anv_state_stream surface_state_stream;
2646 struct anv_state_stream dynamic_state_stream;
2647
2648 VkCommandBufferUsageFlags usage_flags;
2649 VkCommandBufferLevel level;
2650
2651 struct anv_cmd_state state;
2652
2653 /* Set by SetPerformanceMarkerINTEL, written into queries by CmdBeginQuery */
2654 uint64_t intel_perf_marker;
2655 };
2656
2657 VkResult anv_cmd_buffer_init_batch_bo_chain(struct anv_cmd_buffer *cmd_buffer);
2658 void anv_cmd_buffer_fini_batch_bo_chain(struct anv_cmd_buffer *cmd_buffer);
2659 void anv_cmd_buffer_reset_batch_bo_chain(struct anv_cmd_buffer *cmd_buffer);
2660 void anv_cmd_buffer_end_batch_buffer(struct anv_cmd_buffer *cmd_buffer);
2661 void anv_cmd_buffer_add_secondary(struct anv_cmd_buffer *primary,
2662 struct anv_cmd_buffer *secondary);
2663 void anv_cmd_buffer_prepare_execbuf(struct anv_cmd_buffer *cmd_buffer);
2664 VkResult anv_cmd_buffer_execbuf(struct anv_queue *queue,
2665 struct anv_cmd_buffer *cmd_buffer,
2666 const VkSemaphore *in_semaphores,
2667 const uint64_t *in_wait_values,
2668 uint32_t num_in_semaphores,
2669 const VkSemaphore *out_semaphores,
2670 const uint64_t *out_signal_values,
2671 uint32_t num_out_semaphores,
2672 VkFence fence);
2673
2674 VkResult anv_cmd_buffer_reset(struct anv_cmd_buffer *cmd_buffer);
2675
2676 struct anv_state anv_cmd_buffer_emit_dynamic(struct anv_cmd_buffer *cmd_buffer,
2677 const void *data, uint32_t size, uint32_t alignment);
2678 struct anv_state anv_cmd_buffer_merge_dynamic(struct anv_cmd_buffer *cmd_buffer,
2679 uint32_t *a, uint32_t *b,
2680 uint32_t dwords, uint32_t alignment);
2681
2682 struct anv_address
2683 anv_cmd_buffer_surface_base_address(struct anv_cmd_buffer *cmd_buffer);
2684 struct anv_state
2685 anv_cmd_buffer_alloc_binding_table(struct anv_cmd_buffer *cmd_buffer,
2686 uint32_t entries, uint32_t *state_offset);
2687 struct anv_state
2688 anv_cmd_buffer_alloc_surface_state(struct anv_cmd_buffer *cmd_buffer);
2689 struct anv_state
2690 anv_cmd_buffer_alloc_dynamic_state(struct anv_cmd_buffer *cmd_buffer,
2691 uint32_t size, uint32_t alignment);
2692
2693 VkResult
2694 anv_cmd_buffer_new_binding_table_block(struct anv_cmd_buffer *cmd_buffer);
2695
2696 void gen8_cmd_buffer_emit_viewport(struct anv_cmd_buffer *cmd_buffer);
2697 void gen8_cmd_buffer_emit_depth_viewport(struct anv_cmd_buffer *cmd_buffer,
2698 bool depth_clamp_enable);
2699 void gen7_cmd_buffer_emit_scissor(struct anv_cmd_buffer *cmd_buffer);
2700
2701 void anv_cmd_buffer_setup_attachments(struct anv_cmd_buffer *cmd_buffer,
2702 struct anv_render_pass *pass,
2703 struct anv_framebuffer *framebuffer,
2704 const VkClearValue *clear_values);
2705
2706 void anv_cmd_buffer_emit_state_base_address(struct anv_cmd_buffer *cmd_buffer);
2707
2708 struct anv_state
2709 anv_cmd_buffer_push_constants(struct anv_cmd_buffer *cmd_buffer,
2710 gl_shader_stage stage);
2711 struct anv_state
2712 anv_cmd_buffer_cs_push_constants(struct anv_cmd_buffer *cmd_buffer);
2713
2714 const struct anv_image_view *
2715 anv_cmd_buffer_get_depth_stencil_view(const struct anv_cmd_buffer *cmd_buffer);
2716
2717 VkResult
2718 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer,
2719 uint32_t num_entries,
2720 uint32_t *state_offset,
2721 struct anv_state *bt_state);
2722
2723 void anv_cmd_buffer_dump(struct anv_cmd_buffer *cmd_buffer);
2724
2725 void anv_cmd_emit_conditional_render_predicate(struct anv_cmd_buffer *cmd_buffer);
2726
2727 enum anv_fence_type {
2728 ANV_FENCE_TYPE_NONE = 0,
2729 ANV_FENCE_TYPE_BO,
2730 ANV_FENCE_TYPE_SYNCOBJ,
2731 ANV_FENCE_TYPE_WSI,
2732 };
2733
2734 enum anv_bo_fence_state {
2735 /** Indicates that this is a new (or newly reset fence) */
2736 ANV_BO_FENCE_STATE_RESET,
2737
2738 /** Indicates that this fence has been submitted to the GPU but is still
2739 * (as far as we know) in use by the GPU.
2740 */
2741 ANV_BO_FENCE_STATE_SUBMITTED,
2742
2743 ANV_BO_FENCE_STATE_SIGNALED,
2744 };
2745
2746 struct anv_fence_impl {
2747 enum anv_fence_type type;
2748
2749 union {
2750 /** Fence implementation for BO fences
2751 *
2752 * These fences use a BO and a set of CPU-tracked state flags. The BO
2753 * is added to the object list of the last execbuf call in a QueueSubmit
2754 * and is marked EXEC_WRITE. The state flags track when the BO has been
2755 * submitted to the kernel. We need to do this because Vulkan lets you
2756 * wait on a fence that has not yet been submitted and I915_GEM_BUSY
2757 * will say it's idle in this case.
2758 */
2759 struct {
2760 struct anv_bo *bo;
2761 enum anv_bo_fence_state state;
2762 } bo;
2763
2764 /** DRM syncobj handle for syncobj-based fences */
2765 uint32_t syncobj;
2766
2767 /** WSI fence */
2768 struct wsi_fence *fence_wsi;
2769 };
2770 };
2771
2772 struct anv_fence {
2773 /* Permanent fence state. Every fence has some form of permanent state
2774 * (type != ANV_SEMAPHORE_TYPE_NONE). This may be a BO to fence on (for
2775 * cross-process fences) or it could just be a dummy for use internally.
2776 */
2777 struct anv_fence_impl permanent;
2778
2779 /* Temporary fence state. A fence *may* have temporary state. That state
2780 * is added to the fence by an import operation and is reset back to
2781 * ANV_SEMAPHORE_TYPE_NONE when the fence is reset. A fence with temporary
2782 * state cannot be signaled because the fence must already be signaled
2783 * before the temporary state can be exported from the fence in the other
2784 * process and imported here.
2785 */
2786 struct anv_fence_impl temporary;
2787 };
2788
2789 struct anv_event {
2790 uint64_t semaphore;
2791 struct anv_state state;
2792 };
2793
2794 enum anv_semaphore_type {
2795 ANV_SEMAPHORE_TYPE_NONE = 0,
2796 ANV_SEMAPHORE_TYPE_DUMMY,
2797 ANV_SEMAPHORE_TYPE_BO,
2798 ANV_SEMAPHORE_TYPE_SYNC_FILE,
2799 ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ,
2800 ANV_SEMAPHORE_TYPE_TIMELINE,
2801 };
2802
2803 struct anv_timeline_point {
2804 struct list_head link;
2805
2806 uint64_t serial;
2807
2808 /* Number of waiter on this point, when > 0 the point should not be garbage
2809 * collected.
2810 */
2811 int waiting;
2812
2813 /* BO used for synchronization. */
2814 struct anv_bo *bo;
2815 };
2816
2817 struct anv_timeline {
2818 pthread_mutex_t mutex;
2819 pthread_cond_t cond;
2820
2821 uint64_t highest_past;
2822 uint64_t highest_pending;
2823
2824 struct list_head points;
2825 struct list_head free_points;
2826 };
2827
2828 struct anv_semaphore_impl {
2829 enum anv_semaphore_type type;
2830
2831 union {
2832 /* A BO representing this semaphore when type == ANV_SEMAPHORE_TYPE_BO.
2833 * This BO will be added to the object list on any execbuf2 calls for
2834 * which this semaphore is used as a wait or signal fence. When used as
2835 * a signal fence, the EXEC_OBJECT_WRITE flag will be set.
2836 */
2837 struct anv_bo *bo;
2838
2839 /* The sync file descriptor when type == ANV_SEMAPHORE_TYPE_SYNC_FILE.
2840 * If the semaphore is in the unsignaled state due to either just being
2841 * created or because it has been used for a wait, fd will be -1.
2842 */
2843 int fd;
2844
2845 /* Sync object handle when type == ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ.
2846 * Unlike GEM BOs, DRM sync objects aren't deduplicated by the kernel on
2847 * import so we don't need to bother with a userspace cache.
2848 */
2849 uint32_t syncobj;
2850
2851 /* Non shareable timeline semaphore
2852 *
2853 * Used when kernel don't have support for timeline semaphores.
2854 */
2855 struct anv_timeline timeline;
2856 };
2857 };
2858
2859 struct anv_semaphore {
2860 uint32_t refcount;
2861
2862 /* Permanent semaphore state. Every semaphore has some form of permanent
2863 * state (type != ANV_SEMAPHORE_TYPE_NONE). This may be a BO to fence on
2864 * (for cross-process semaphores0 or it could just be a dummy for use
2865 * internally.
2866 */
2867 struct anv_semaphore_impl permanent;
2868
2869 /* Temporary semaphore state. A semaphore *may* have temporary state.
2870 * That state is added to the semaphore by an import operation and is reset
2871 * back to ANV_SEMAPHORE_TYPE_NONE when the semaphore is waited on. A
2872 * semaphore with temporary state cannot be signaled because the semaphore
2873 * must already be signaled before the temporary state can be exported from
2874 * the semaphore in the other process and imported here.
2875 */
2876 struct anv_semaphore_impl temporary;
2877 };
2878
2879 void anv_semaphore_reset_temporary(struct anv_device *device,
2880 struct anv_semaphore *semaphore);
2881
2882 struct anv_shader_module {
2883 unsigned char sha1[20];
2884 uint32_t size;
2885 char data[0];
2886 };
2887
2888 static inline gl_shader_stage
2889 vk_to_mesa_shader_stage(VkShaderStageFlagBits vk_stage)
2890 {
2891 assert(__builtin_popcount(vk_stage) == 1);
2892 return ffs(vk_stage) - 1;
2893 }
2894
2895 static inline VkShaderStageFlagBits
2896 mesa_to_vk_shader_stage(gl_shader_stage mesa_stage)
2897 {
2898 return (1 << mesa_stage);
2899 }
2900
2901 #define ANV_STAGE_MASK ((1 << MESA_SHADER_STAGES) - 1)
2902
2903 #define anv_foreach_stage(stage, stage_bits) \
2904 for (gl_shader_stage stage, \
2905 __tmp = (gl_shader_stage)((stage_bits) & ANV_STAGE_MASK); \
2906 stage = __builtin_ffs(__tmp) - 1, __tmp; \
2907 __tmp &= ~(1 << (stage)))
2908
2909 struct anv_pipeline_bind_map {
2910 uint32_t surface_count;
2911 uint32_t sampler_count;
2912
2913 struct anv_pipeline_binding * surface_to_descriptor;
2914 struct anv_pipeline_binding * sampler_to_descriptor;
2915 };
2916
2917 struct anv_shader_bin_key {
2918 uint32_t size;
2919 uint8_t data[0];
2920 };
2921
2922 struct anv_shader_bin {
2923 uint32_t ref_cnt;
2924
2925 const struct anv_shader_bin_key *key;
2926
2927 struct anv_state kernel;
2928 uint32_t kernel_size;
2929
2930 struct anv_state constant_data;
2931 uint32_t constant_data_size;
2932
2933 const struct brw_stage_prog_data *prog_data;
2934 uint32_t prog_data_size;
2935
2936 struct brw_compile_stats stats[3];
2937 uint32_t num_stats;
2938
2939 struct nir_xfb_info *xfb_info;
2940
2941 struct anv_pipeline_bind_map bind_map;
2942 };
2943
2944 struct anv_shader_bin *
2945 anv_shader_bin_create(struct anv_device *device,
2946 const void *key, uint32_t key_size,
2947 const void *kernel, uint32_t kernel_size,
2948 const void *constant_data, uint32_t constant_data_size,
2949 const struct brw_stage_prog_data *prog_data,
2950 uint32_t prog_data_size, const void *prog_data_param,
2951 const struct brw_compile_stats *stats, uint32_t num_stats,
2952 const struct nir_xfb_info *xfb_info,
2953 const struct anv_pipeline_bind_map *bind_map);
2954
2955 void
2956 anv_shader_bin_destroy(struct anv_device *device, struct anv_shader_bin *shader);
2957
2958 static inline void
2959 anv_shader_bin_ref(struct anv_shader_bin *shader)
2960 {
2961 assert(shader && shader->ref_cnt >= 1);
2962 p_atomic_inc(&shader->ref_cnt);
2963 }
2964
2965 static inline void
2966 anv_shader_bin_unref(struct anv_device *device, struct anv_shader_bin *shader)
2967 {
2968 assert(shader && shader->ref_cnt >= 1);
2969 if (p_atomic_dec_zero(&shader->ref_cnt))
2970 anv_shader_bin_destroy(device, shader);
2971 }
2972
2973 /* 5 possible simultaneous shader stages and FS may have up to 3 binaries */
2974 #define MAX_PIPELINE_EXECUTABLES 7
2975
2976 struct anv_pipeline_executable {
2977 gl_shader_stage stage;
2978
2979 struct brw_compile_stats stats;
2980
2981 char *nir;
2982 char *disasm;
2983 };
2984
2985 struct anv_pipeline {
2986 struct anv_device * device;
2987 struct anv_batch batch;
2988 uint32_t batch_data[512];
2989 struct anv_reloc_list batch_relocs;
2990 anv_cmd_dirty_mask_t dynamic_state_mask;
2991 struct anv_dynamic_state dynamic_state;
2992
2993 void * mem_ctx;
2994
2995 VkPipelineCreateFlags flags;
2996 struct anv_subpass * subpass;
2997
2998 bool needs_data_cache;
2999
3000 struct anv_shader_bin * shaders[MESA_SHADER_STAGES];
3001
3002 uint32_t num_executables;
3003 struct anv_pipeline_executable executables[MAX_PIPELINE_EXECUTABLES];
3004
3005 struct {
3006 const struct gen_l3_config * l3_config;
3007 uint32_t total_size;
3008 } urb;
3009
3010 VkShaderStageFlags active_stages;
3011 struct anv_state blend_state;
3012
3013 uint32_t vb_used;
3014 struct anv_pipeline_vertex_binding {
3015 uint32_t stride;
3016 bool instanced;
3017 uint32_t instance_divisor;
3018 } vb[MAX_VBS];
3019
3020 uint8_t xfb_used;
3021
3022 bool primitive_restart;
3023 uint32_t topology;
3024
3025 uint32_t cs_right_mask;
3026
3027 bool writes_depth;
3028 bool depth_test_enable;
3029 bool writes_stencil;
3030 bool stencil_test_enable;
3031 bool depth_clamp_enable;
3032 bool depth_clip_enable;
3033 bool sample_shading_enable;
3034 bool kill_pixel;
3035 bool depth_bounds_test_enable;
3036
3037 struct {
3038 uint32_t sf[7];
3039 uint32_t depth_stencil_state[3];
3040 } gen7;
3041
3042 struct {
3043 uint32_t sf[4];
3044 uint32_t raster[5];
3045 uint32_t wm_depth_stencil[3];
3046 } gen8;
3047
3048 struct {
3049 uint32_t wm_depth_stencil[4];
3050 } gen9;
3051
3052 uint32_t interface_descriptor_data[8];
3053 };
3054
3055 static inline bool
3056 anv_pipeline_has_stage(const struct anv_pipeline *pipeline,
3057 gl_shader_stage stage)
3058 {
3059 return (pipeline->active_stages & mesa_to_vk_shader_stage(stage)) != 0;
3060 }
3061
3062 #define ANV_DECL_GET_PROG_DATA_FUNC(prefix, stage) \
3063 static inline const struct brw_##prefix##_prog_data * \
3064 get_##prefix##_prog_data(const struct anv_pipeline *pipeline) \
3065 { \
3066 if (anv_pipeline_has_stage(pipeline, stage)) { \
3067 return (const struct brw_##prefix##_prog_data *) \
3068 pipeline->shaders[stage]->prog_data; \
3069 } else { \
3070 return NULL; \
3071 } \
3072 }
3073
3074 ANV_DECL_GET_PROG_DATA_FUNC(vs, MESA_SHADER_VERTEX)
3075 ANV_DECL_GET_PROG_DATA_FUNC(tcs, MESA_SHADER_TESS_CTRL)
3076 ANV_DECL_GET_PROG_DATA_FUNC(tes, MESA_SHADER_TESS_EVAL)
3077 ANV_DECL_GET_PROG_DATA_FUNC(gs, MESA_SHADER_GEOMETRY)
3078 ANV_DECL_GET_PROG_DATA_FUNC(wm, MESA_SHADER_FRAGMENT)
3079 ANV_DECL_GET_PROG_DATA_FUNC(cs, MESA_SHADER_COMPUTE)
3080
3081 static inline const struct brw_vue_prog_data *
3082 anv_pipeline_get_last_vue_prog_data(const struct anv_pipeline *pipeline)
3083 {
3084 if (anv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY))
3085 return &get_gs_prog_data(pipeline)->base;
3086 else if (anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL))
3087 return &get_tes_prog_data(pipeline)->base;
3088 else
3089 return &get_vs_prog_data(pipeline)->base;
3090 }
3091
3092 VkResult
3093 anv_pipeline_init(struct anv_pipeline *pipeline, struct anv_device *device,
3094 struct anv_pipeline_cache *cache,
3095 const VkGraphicsPipelineCreateInfo *pCreateInfo,
3096 const VkAllocationCallbacks *alloc);
3097
3098 VkResult
3099 anv_pipeline_compile_cs(struct anv_pipeline *pipeline,
3100 struct anv_pipeline_cache *cache,
3101 const VkComputePipelineCreateInfo *info,
3102 const struct anv_shader_module *module,
3103 const char *entrypoint,
3104 const VkSpecializationInfo *spec_info);
3105
3106 struct anv_format_plane {
3107 enum isl_format isl_format:16;
3108 struct isl_swizzle swizzle;
3109
3110 /* Whether this plane contains chroma channels */
3111 bool has_chroma;
3112
3113 /* For downscaling of YUV planes */
3114 uint8_t denominator_scales[2];
3115
3116 /* How to map sampled ycbcr planes to a single 4 component element. */
3117 struct isl_swizzle ycbcr_swizzle;
3118
3119 /* What aspect is associated to this plane */
3120 VkImageAspectFlags aspect;
3121 };
3122
3123
3124 struct anv_format {
3125 struct anv_format_plane planes[3];
3126 VkFormat vk_format;
3127 uint8_t n_planes;
3128 bool can_ycbcr;
3129 };
3130
3131 static inline uint32_t
3132 anv_image_aspect_to_plane(VkImageAspectFlags image_aspects,
3133 VkImageAspectFlags aspect_mask)
3134 {
3135 switch (aspect_mask) {
3136 case VK_IMAGE_ASPECT_COLOR_BIT:
3137 case VK_IMAGE_ASPECT_DEPTH_BIT:
3138 case VK_IMAGE_ASPECT_PLANE_0_BIT:
3139 return 0;
3140 case VK_IMAGE_ASPECT_STENCIL_BIT:
3141 if ((image_aspects & VK_IMAGE_ASPECT_DEPTH_BIT) == 0)
3142 return 0;
3143 /* Fall-through */
3144 case VK_IMAGE_ASPECT_PLANE_1_BIT:
3145 return 1;
3146 case VK_IMAGE_ASPECT_PLANE_2_BIT:
3147 return 2;
3148 default:
3149 /* Purposefully assert with depth/stencil aspects. */
3150 unreachable("invalid image aspect");
3151 }
3152 }
3153
3154 static inline VkImageAspectFlags
3155 anv_plane_to_aspect(VkImageAspectFlags image_aspects,
3156 uint32_t plane)
3157 {
3158 if (image_aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) {
3159 if (util_bitcount(image_aspects) > 1)
3160 return VK_IMAGE_ASPECT_PLANE_0_BIT << plane;
3161 return VK_IMAGE_ASPECT_COLOR_BIT;
3162 }
3163 if (image_aspects & VK_IMAGE_ASPECT_DEPTH_BIT)
3164 return VK_IMAGE_ASPECT_DEPTH_BIT << plane;
3165 assert(image_aspects == VK_IMAGE_ASPECT_STENCIL_BIT);
3166 return VK_IMAGE_ASPECT_STENCIL_BIT;
3167 }
3168
3169 #define anv_foreach_image_aspect_bit(b, image, aspects) \
3170 for_each_bit(b, anv_image_expand_aspects(image, aspects))
3171
3172 const struct anv_format *
3173 anv_get_format(VkFormat format);
3174
3175 static inline uint32_t
3176 anv_get_format_planes(VkFormat vk_format)
3177 {
3178 const struct anv_format *format = anv_get_format(vk_format);
3179
3180 return format != NULL ? format->n_planes : 0;
3181 }
3182
3183 struct anv_format_plane
3184 anv_get_format_plane(const struct gen_device_info *devinfo, VkFormat vk_format,
3185 VkImageAspectFlagBits aspect, VkImageTiling tiling);
3186
3187 static inline enum isl_format
3188 anv_get_isl_format(const struct gen_device_info *devinfo, VkFormat vk_format,
3189 VkImageAspectFlags aspect, VkImageTiling tiling)
3190 {
3191 return anv_get_format_plane(devinfo, vk_format, aspect, tiling).isl_format;
3192 }
3193
3194 static inline struct isl_swizzle
3195 anv_swizzle_for_render(struct isl_swizzle swizzle)
3196 {
3197 /* Sometimes the swizzle will have alpha map to one. We do this to fake
3198 * RGB as RGBA for texturing
3199 */
3200 assert(swizzle.a == ISL_CHANNEL_SELECT_ONE ||
3201 swizzle.a == ISL_CHANNEL_SELECT_ALPHA);
3202
3203 /* But it doesn't matter what we render to that channel */
3204 swizzle.a = ISL_CHANNEL_SELECT_ALPHA;
3205
3206 return swizzle;
3207 }
3208
3209 void
3210 anv_pipeline_setup_l3_config(struct anv_pipeline *pipeline, bool needs_slm);
3211
3212 /**
3213 * Subsurface of an anv_image.
3214 */
3215 struct anv_surface {
3216 /** Valid only if isl_surf::size_B > 0. */
3217 struct isl_surf isl;
3218
3219 /**
3220 * Offset from VkImage's base address, as bound by vkBindImageMemory().
3221 */
3222 uint32_t offset;
3223 };
3224
3225 struct anv_image {
3226 VkImageType type; /**< VkImageCreateInfo::imageType */
3227 /* The original VkFormat provided by the client. This may not match any
3228 * of the actual surface formats.
3229 */
3230 VkFormat vk_format;
3231 const struct anv_format *format;
3232
3233 VkImageAspectFlags aspects;
3234 VkExtent3D extent;
3235 uint32_t levels;
3236 uint32_t array_size;
3237 uint32_t samples; /**< VkImageCreateInfo::samples */
3238 uint32_t n_planes;
3239 VkImageUsageFlags usage; /**< VkImageCreateInfo::usage. */
3240 VkImageUsageFlags stencil_usage;
3241 VkImageCreateFlags create_flags; /* Flags used when creating image. */
3242 VkImageTiling tiling; /** VkImageCreateInfo::tiling */
3243
3244 /** True if this is needs to be bound to an appropriately tiled BO.
3245 *
3246 * When not using modifiers, consumers such as X11, Wayland, and KMS need
3247 * the tiling passed via I915_GEM_SET_TILING. When exporting these buffers
3248 * we require a dedicated allocation so that we can know to allocate a
3249 * tiled buffer.
3250 */
3251 bool needs_set_tiling;
3252
3253 /**
3254 * Must be DRM_FORMAT_MOD_INVALID unless tiling is
3255 * VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT.
3256 */
3257 uint64_t drm_format_mod;
3258
3259 VkDeviceSize size;
3260 uint32_t alignment;
3261
3262 /* Whether the image is made of several underlying buffer objects rather a
3263 * single one with different offsets.
3264 */
3265 bool disjoint;
3266
3267 /* All the formats that can be used when creating views of this image
3268 * are CCS_E compatible.
3269 */
3270 bool ccs_e_compatible;
3271
3272 /* Image was created with external format. */
3273 bool external_format;
3274
3275 /**
3276 * Image subsurfaces
3277 *
3278 * For each foo, anv_image::planes[x].surface is valid if and only if
3279 * anv_image::aspects has a x aspect. Refer to anv_image_aspect_to_plane()
3280 * to figure the number associated with a given aspect.
3281 *
3282 * The hardware requires that the depth buffer and stencil buffer be
3283 * separate surfaces. From Vulkan's perspective, though, depth and stencil
3284 * reside in the same VkImage. To satisfy both the hardware and Vulkan, we
3285 * allocate the depth and stencil buffers as separate surfaces in the same
3286 * bo.
3287 *
3288 * Memory layout :
3289 *
3290 * -----------------------
3291 * | surface0 | /|\
3292 * ----------------------- |
3293 * | shadow surface0 | |
3294 * ----------------------- | Plane 0
3295 * | aux surface0 | |
3296 * ----------------------- |
3297 * | fast clear colors0 | \|/
3298 * -----------------------
3299 * | surface1 | /|\
3300 * ----------------------- |
3301 * | shadow surface1 | |
3302 * ----------------------- | Plane 1
3303 * | aux surface1 | |
3304 * ----------------------- |
3305 * | fast clear colors1 | \|/
3306 * -----------------------
3307 * | ... |
3308 * | |
3309 * -----------------------
3310 */
3311 struct {
3312 /**
3313 * Offset of the entire plane (whenever the image is disjoint this is
3314 * set to 0).
3315 */
3316 uint32_t offset;
3317
3318 VkDeviceSize size;
3319 uint32_t alignment;
3320
3321 struct anv_surface surface;
3322
3323 /**
3324 * A surface which shadows the main surface and may have different
3325 * tiling. This is used for sampling using a tiling that isn't supported
3326 * for other operations.
3327 */
3328 struct anv_surface shadow_surface;
3329
3330 /**
3331 * For color images, this is the aux usage for this image when not used
3332 * as a color attachment.
3333 *
3334 * For depth/stencil images, this is set to ISL_AUX_USAGE_HIZ if the
3335 * image has a HiZ buffer.
3336 */
3337 enum isl_aux_usage aux_usage;
3338
3339 struct anv_surface aux_surface;
3340
3341 /**
3342 * Offset of the fast clear state (used to compute the
3343 * fast_clear_state_offset of the following planes).
3344 */
3345 uint32_t fast_clear_state_offset;
3346
3347 /**
3348 * BO associated with this plane, set when bound.
3349 */
3350 struct anv_address address;
3351
3352 /**
3353 * Address of the main surface used to fill the aux map table. This is
3354 * used at destruction of the image since the Vulkan spec does not
3355 * guarantee that the address.bo field we still be valid at destruction.
3356 */
3357 uint64_t aux_map_surface_address;
3358
3359 /**
3360 * When destroying the image, also free the bo.
3361 * */
3362 bool bo_is_owned;
3363 } planes[3];
3364 };
3365
3366 /* The ordering of this enum is important */
3367 enum anv_fast_clear_type {
3368 /** Image does not have/support any fast-clear blocks */
3369 ANV_FAST_CLEAR_NONE = 0,
3370 /** Image has/supports fast-clear but only to the default value */
3371 ANV_FAST_CLEAR_DEFAULT_VALUE = 1,
3372 /** Image has/supports fast-clear with an arbitrary fast-clear value */
3373 ANV_FAST_CLEAR_ANY = 2,
3374 };
3375
3376 /* Returns the number of auxiliary buffer levels attached to an image. */
3377 static inline uint8_t
3378 anv_image_aux_levels(const struct anv_image * const image,
3379 VkImageAspectFlagBits aspect)
3380 {
3381 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
3382
3383 /* The Gen12 CCS aux surface is represented with only one level. */
3384 const uint8_t aux_logical_levels =
3385 image->planes[plane].aux_surface.isl.tiling == ISL_TILING_GEN12_CCS ?
3386 image->planes[plane].surface.isl.levels :
3387 image->planes[plane].aux_surface.isl.levels;
3388
3389 return image->planes[plane].aux_surface.isl.size_B > 0 ?
3390 aux_logical_levels : 0;
3391 }
3392
3393 /* Returns the number of auxiliary buffer layers attached to an image. */
3394 static inline uint32_t
3395 anv_image_aux_layers(const struct anv_image * const image,
3396 VkImageAspectFlagBits aspect,
3397 const uint8_t miplevel)
3398 {
3399 assert(image);
3400
3401 /* The miplevel must exist in the main buffer. */
3402 assert(miplevel < image->levels);
3403
3404 if (miplevel >= anv_image_aux_levels(image, aspect)) {
3405 /* There are no layers with auxiliary data because the miplevel has no
3406 * auxiliary data.
3407 */
3408 return 0;
3409 } else {
3410 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
3411
3412 /* The Gen12 CCS aux surface is represented with only one layer. */
3413 const struct isl_extent4d *aux_logical_level0_px =
3414 image->planes[plane].aux_surface.isl.tiling == ISL_TILING_GEN12_CCS ?
3415 &image->planes[plane].surface.isl.logical_level0_px :
3416 &image->planes[plane].aux_surface.isl.logical_level0_px;
3417
3418 return MAX2(aux_logical_level0_px->array_len,
3419 aux_logical_level0_px->depth >> miplevel);
3420 }
3421 }
3422
3423 static inline struct anv_address
3424 anv_image_get_clear_color_addr(const struct anv_device *device,
3425 const struct anv_image *image,
3426 VkImageAspectFlagBits aspect)
3427 {
3428 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV);
3429
3430 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect);
3431 return anv_address_add(image->planes[plane].address,
3432 image->planes[plane].fast_clear_state_offset);
3433 }
3434
3435 static inline struct anv_address