2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
30 #include "anv_private.h"
32 #include "common/gen_sample_positions.h"
33 #include "genxml/gen_macros.h"
34 #include "genxml/genX_pack.h"
40 * From Gen10 Workarounds page in h/w specs:
42 * "Prior to the 3DSTATE_SAMPLE_PATTERN driver must ensure there are no
43 * markers in the pipeline by programming a PIPE_CONTROL with stall."
46 gen10_emit_wa_cs_stall_flush(struct anv_batch
*batch
)
49 anv_batch_emit(batch
, GENX(PIPE_CONTROL
), pc
) {
50 pc
.CommandStreamerStallEnable
= true;
51 pc
.StallAtPixelScoreboard
= true;
56 * From Gen10 Workarounds page in h/w specs:
57 * WaSampleOffsetIZ:_cs_stall_flush
58 * "When 3DSTATE_SAMPLE_PATTERN is programmed, driver must then issue an
59 * MI_LOAD_REGISTER_IMM command to an offset between 0x7000 and 0x7FFF(SVL)
60 * after the command to ensure the state has been delivered prior to any
61 * command causing a marker in the pipeline."
64 gen10_emit_wa_lri_to_cache_mode_zero(struct anv_batch
*batch
)
66 /* Before changing the value of CACHE_MODE_0 register, GFX pipeline must
67 * be idle; i.e., full flush is required.
69 anv_batch_emit(batch
, GENX(PIPE_CONTROL
), pc
) {
70 pc
.DepthCacheFlushEnable
= true;
71 pc
.DCFlushEnable
= true;
72 pc
.RenderTargetCacheFlushEnable
= true;
73 pc
.InstructionCacheInvalidateEnable
= true;
74 pc
.StateCacheInvalidationEnable
= true;
75 pc
.TextureCacheInvalidationEnable
= true;
76 pc
.VFCacheInvalidationEnable
= true;
77 pc
.ConstantCacheInvalidationEnable
=true;
80 /* Write to CACHE_MODE_0 (0x7000) */
81 uint32_t cache_mode_0
= 0;
82 anv_pack_struct(&cache_mode_0
, GENX(CACHE_MODE_0
));
84 anv_batch_emit(batch
, GENX(MI_LOAD_REGISTER_IMM
), lri
) {
85 lri
.RegisterOffset
= GENX(CACHE_MODE_0_num
);
86 lri
.DataDWord
= cache_mode_0
;
92 genX(init_device_state
)(struct anv_device
*device
)
94 device
->default_mocs
= GENX(MOCS
);
96 device
->external_mocs
= GENX(EXTERNAL_MOCS
);
98 device
->external_mocs
= device
->default_mocs
;
101 struct anv_batch batch
;
104 batch
.start
= batch
.next
= cmds
;
105 batch
.end
= (void *) cmds
+ sizeof(cmds
);
107 anv_batch_emit(&batch
, GENX(PIPELINE_SELECT
), ps
) {
111 ps
.PipelineSelection
= _3D
;
115 uint32_t cache_mode_1
;
116 anv_pack_struct(&cache_mode_1
, GENX(CACHE_MODE_1
),
117 .FloatBlendOptimizationEnable
= true,
118 .FloatBlendOptimizationEnableMask
= true,
119 .PartialResolveDisableInVC
= true,
120 .PartialResolveDisableInVCMask
= true);
122 anv_batch_emit(&batch
, GENX(MI_LOAD_REGISTER_IMM
), lri
) {
123 lri
.RegisterOffset
= GENX(CACHE_MODE_1_num
);
124 lri
.DataDWord
= cache_mode_1
;
128 anv_batch_emit(&batch
, GENX(3DSTATE_AA_LINE_PARAMETERS
), aa
);
130 anv_batch_emit(&batch
, GENX(3DSTATE_DRAWING_RECTANGLE
), rect
) {
131 rect
.ClippedDrawingRectangleYMin
= 0;
132 rect
.ClippedDrawingRectangleXMin
= 0;
133 rect
.ClippedDrawingRectangleYMax
= UINT16_MAX
;
134 rect
.ClippedDrawingRectangleXMax
= UINT16_MAX
;
135 rect
.DrawingRectangleOriginY
= 0;
136 rect
.DrawingRectangleOriginX
= 0;
140 anv_batch_emit(&batch
, GENX(3DSTATE_WM_CHROMAKEY
), ck
);
143 gen10_emit_wa_cs_stall_flush(&batch
);
146 /* See the Vulkan 1.0 spec Table 24.1 "Standard sample locations" and
147 * VkPhysicalDeviceFeatures::standardSampleLocations.
149 anv_batch_emit(&batch
, GENX(3DSTATE_SAMPLE_PATTERN
), sp
) {
150 GEN_SAMPLE_POS_1X(sp
._1xSample
);
151 GEN_SAMPLE_POS_2X(sp
._2xSample
);
152 GEN_SAMPLE_POS_4X(sp
._4xSample
);
153 GEN_SAMPLE_POS_8X(sp
._8xSample
);
155 GEN_SAMPLE_POS_16X(sp
._16xSample
);
159 /* The BDW+ docs describe how to use the 3DSTATE_WM_HZ_OP instruction in the
160 * section titled, "Optimized Depth Buffer Clear and/or Stencil Buffer
161 * Clear." It mentions that the packet overrides GPU state for the clear
162 * operation and needs to be reset to 0s to clear the overrides. Depending
163 * on the kernel, we may not get a context with the state for this packet
164 * zeroed. Do it ourselves just in case. We've observed this to prevent a
165 * number of GPU hangs on ICL.
167 anv_batch_emit(&batch
, GENX(3DSTATE_WM_HZ_OP
), hzp
);
171 gen10_emit_wa_lri_to_cache_mode_zero(&batch
);
175 /* The default behavior of bit 5 "Headerless Message for Pre-emptable
176 * Contexts" in SAMPLER MODE register is set to 0, which means
177 * headerless sampler messages are not allowed for pre-emptable
178 * contexts. Set the bit 5 to 1 to allow them.
180 uint32_t sampler_mode
;
181 anv_pack_struct(&sampler_mode
, GENX(SAMPLER_MODE
),
182 .HeaderlessMessageforPreemptableContexts
= true,
183 .HeaderlessMessageforPreemptableContextsMask
= true);
185 anv_batch_emit(&batch
, GENX(MI_LOAD_REGISTER_IMM
), lri
) {
186 lri
.RegisterOffset
= GENX(SAMPLER_MODE_num
);
187 lri
.DataDWord
= sampler_mode
;
190 /* Bit 1 "Enabled Texel Offset Precision Fix" must be set in
191 * HALF_SLICE_CHICKEN7 register.
193 uint32_t half_slice_chicken7
;
194 anv_pack_struct(&half_slice_chicken7
, GENX(HALF_SLICE_CHICKEN7
),
195 .EnabledTexelOffsetPrecisionFix
= true,
196 .EnabledTexelOffsetPrecisionFixMask
= true);
198 anv_batch_emit(&batch
, GENX(MI_LOAD_REGISTER_IMM
), lri
) {
199 lri
.RegisterOffset
= GENX(HALF_SLICE_CHICKEN7_num
);
200 lri
.DataDWord
= half_slice_chicken7
;
203 /* WaEnableStateCacheRedirectToCS:icl */
204 uint32_t slice_common_eco_chicken1
;
205 anv_pack_struct(&slice_common_eco_chicken1
,
206 GENX(SLICE_COMMON_ECO_CHICKEN1
),
207 .StateCacheRedirectToCSSectionEnable
= true,
208 .StateCacheRedirectToCSSectionEnableMask
= true);
210 anv_batch_emit(&batch
, GENX(MI_LOAD_REGISTER_IMM
), lri
) {
211 lri
.RegisterOffset
= GENX(SLICE_COMMON_ECO_CHICKEN1_num
);
212 lri
.DataDWord
= slice_common_eco_chicken1
;
217 /* hardware specification recommends disabling repacking for
218 * the compatibility with decompression mechanism in display controller.
220 if (device
->info
.disable_ccs_repack
) {
221 uint32_t cache_mode_0
;
222 anv_pack_struct(&cache_mode_0
,
224 .DisableRepackingforCompression
= true,
225 .DisableRepackingforCompressionMask
= true);
227 anv_batch_emit(&batch
, GENX(MI_LOAD_REGISTER_IMM
), lri
) {
228 lri
.RegisterOffset
= GENX(CACHE_MODE_0_num
);
229 lri
.DataDWord
= cache_mode_0
;
234 /* Set the "CONSTANT_BUFFER Address Offset Disable" bit, so
235 * 3DSTATE_CONSTANT_XS buffer 0 is an absolute address.
237 * This is only safe on kernels with context isolation support.
240 device
->instance
->physicalDevice
.has_context_isolation
) {
241 UNUSED
uint32_t tmp_reg
;
243 anv_pack_struct(&tmp_reg
, GENX(CS_DEBUG_MODE2
),
244 .CONSTANT_BUFFERAddressOffsetDisable
= true,
245 .CONSTANT_BUFFERAddressOffsetDisableMask
= true);
246 anv_batch_emit(&batch
, GENX(MI_LOAD_REGISTER_IMM
), lri
) {
247 lri
.RegisterOffset
= GENX(CS_DEBUG_MODE2_num
);
248 lri
.DataDWord
= tmp_reg
;
251 anv_pack_struct(&tmp_reg
, GENX(INSTPM
),
252 .CONSTANT_BUFFERAddressOffsetDisable
= true,
253 .CONSTANT_BUFFERAddressOffsetDisableMask
= true);
254 anv_batch_emit(&batch
, GENX(MI_LOAD_REGISTER_IMM
), lri
) {
255 lri
.RegisterOffset
= GENX(INSTPM_num
);
256 lri
.DataDWord
= tmp_reg
;
261 anv_batch_emit(&batch
, GENX(MI_BATCH_BUFFER_END
), bbe
);
263 assert(batch
.next
<= batch
.end
);
265 return anv_device_submit_simple_batch(device
, &batch
);
269 vk_to_gen_tex_filter(VkFilter filter
, bool anisotropyEnable
)
273 assert(!"Invalid filter");
274 case VK_FILTER_NEAREST
:
275 return anisotropyEnable
? MAPFILTER_ANISOTROPIC
: MAPFILTER_NEAREST
;
276 case VK_FILTER_LINEAR
:
277 return anisotropyEnable
? MAPFILTER_ANISOTROPIC
: MAPFILTER_LINEAR
;
282 vk_to_gen_max_anisotropy(float ratio
)
284 return (anv_clamp_f(ratio
, 2, 16) - 2) / 2;
287 static const uint32_t vk_to_gen_mipmap_mode
[] = {
288 [VK_SAMPLER_MIPMAP_MODE_NEAREST
] = MIPFILTER_NEAREST
,
289 [VK_SAMPLER_MIPMAP_MODE_LINEAR
] = MIPFILTER_LINEAR
292 static const uint32_t vk_to_gen_tex_address
[] = {
293 [VK_SAMPLER_ADDRESS_MODE_REPEAT
] = TCM_WRAP
,
294 [VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT
] = TCM_MIRROR
,
295 [VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE
] = TCM_CLAMP
,
296 [VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE
] = TCM_MIRROR_ONCE
,
297 [VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER
] = TCM_CLAMP_BORDER
,
300 /* Vulkan specifies the result of shadow comparisons as:
301 * 1 if ref <op> texel,
305 * 0 if texel <op> ref,
308 * So, these look a bit strange because there's both a negation
309 * and swapping of the arguments involved.
311 static const uint32_t vk_to_gen_shadow_compare_op
[] = {
312 [VK_COMPARE_OP_NEVER
] = PREFILTEROPALWAYS
,
313 [VK_COMPARE_OP_LESS
] = PREFILTEROPLEQUAL
,
314 [VK_COMPARE_OP_EQUAL
] = PREFILTEROPNOTEQUAL
,
315 [VK_COMPARE_OP_LESS_OR_EQUAL
] = PREFILTEROPLESS
,
316 [VK_COMPARE_OP_GREATER
] = PREFILTEROPGEQUAL
,
317 [VK_COMPARE_OP_NOT_EQUAL
] = PREFILTEROPEQUAL
,
318 [VK_COMPARE_OP_GREATER_OR_EQUAL
] = PREFILTEROPGREATER
,
319 [VK_COMPARE_OP_ALWAYS
] = PREFILTEROPNEVER
,
323 static const uint32_t vk_to_gen_sampler_reduction_mode
[] = {
324 [VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT
] = STD_FILTER
,
325 [VK_SAMPLER_REDUCTION_MODE_MIN_EXT
] = MINIMUM
,
326 [VK_SAMPLER_REDUCTION_MODE_MAX_EXT
] = MAXIMUM
,
330 VkResult
genX(CreateSampler
)(
332 const VkSamplerCreateInfo
* pCreateInfo
,
333 const VkAllocationCallbacks
* pAllocator
,
336 ANV_FROM_HANDLE(anv_device
, device
, _device
);
337 const struct anv_physical_device
*pdevice
=
338 &device
->instance
->physicalDevice
;
339 struct anv_sampler
*sampler
;
341 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO
);
343 sampler
= vk_zalloc2(&device
->alloc
, pAllocator
, sizeof(*sampler
), 8,
344 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
346 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
348 sampler
->n_planes
= 1;
350 uint32_t border_color_stride
= GEN_IS_HASWELL
? 512 : 64;
351 uint32_t border_color_offset
= device
->border_colors
.offset
+
352 pCreateInfo
->borderColor
*
356 unsigned sampler_reduction_mode
= STD_FILTER
;
357 bool enable_sampler_reduction
= false;
360 vk_foreach_struct(ext
, pCreateInfo
->pNext
) {
361 switch (ext
->sType
) {
362 case VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO
: {
363 VkSamplerYcbcrConversionInfo
*pSamplerConversion
=
364 (VkSamplerYcbcrConversionInfo
*) ext
;
365 ANV_FROM_HANDLE(anv_ycbcr_conversion
, conversion
,
366 pSamplerConversion
->conversion
);
368 /* Ignore conversion for non-YUV formats. This fulfills a requirement
369 * for clients that want to utilize same code path for images with
370 * external formats (VK_FORMAT_UNDEFINED) and "regular" RGBA images
371 * where format is known.
373 if (conversion
== NULL
|| !conversion
->format
->can_ycbcr
)
376 sampler
->n_planes
= conversion
->format
->n_planes
;
377 sampler
->conversion
= conversion
;
381 case VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO_EXT
: {
382 struct VkSamplerReductionModeCreateInfoEXT
*sampler_reduction
=
383 (struct VkSamplerReductionModeCreateInfoEXT
*) ext
;
384 sampler_reduction_mode
=
385 vk_to_gen_sampler_reduction_mode
[sampler_reduction
->reductionMode
];
386 enable_sampler_reduction
= true;
391 anv_debug_ignored_stype(ext
->sType
);
396 if (pdevice
->has_bindless_samplers
) {
397 /* If we have bindless, allocate enough samplers. We allocate 32 bytes
398 * for each sampler instead of 16 bytes because we want all bindless
399 * samplers to be 32-byte aligned so we don't have to use indirect
400 * sampler messages on them.
402 sampler
->bindless_state
=
403 anv_state_pool_alloc(&device
->dynamic_state_pool
,
404 sampler
->n_planes
* 32, 32);
407 for (unsigned p
= 0; p
< sampler
->n_planes
; p
++) {
408 const bool plane_has_chroma
=
409 sampler
->conversion
&& sampler
->conversion
->format
->planes
[p
].has_chroma
;
410 const VkFilter min_filter
=
411 plane_has_chroma
? sampler
->conversion
->chroma_filter
: pCreateInfo
->minFilter
;
412 const VkFilter mag_filter
=
413 plane_has_chroma
? sampler
->conversion
->chroma_filter
: pCreateInfo
->magFilter
;
414 const bool enable_min_filter_addr_rounding
= min_filter
!= VK_FILTER_NEAREST
;
415 const bool enable_mag_filter_addr_rounding
= mag_filter
!= VK_FILTER_NEAREST
;
416 /* From Broadwell PRM, SAMPLER_STATE:
417 * "Mip Mode Filter must be set to MIPFILTER_NONE for Planar YUV surfaces."
419 const uint32_t mip_filter_mode
=
420 (sampler
->conversion
&&
421 isl_format_is_yuv(sampler
->conversion
->format
->planes
[0].isl_format
)) ?
422 MIPFILTER_NONE
: vk_to_gen_mipmap_mode
[pCreateInfo
->mipmapMode
];
424 struct GENX(SAMPLER_STATE
) sampler_state
= {
425 .SamplerDisable
= false,
426 .TextureBorderColorMode
= DX10OGL
,
429 .LODPreClampMode
= CLAMP_MODE_OGL
,
431 .LODPreClampEnable
= CLAMP_ENABLE_OGL
,
437 .MipModeFilter
= mip_filter_mode
,
438 .MagModeFilter
= vk_to_gen_tex_filter(mag_filter
, pCreateInfo
->anisotropyEnable
),
439 .MinModeFilter
= vk_to_gen_tex_filter(min_filter
, pCreateInfo
->anisotropyEnable
),
440 .TextureLODBias
= anv_clamp_f(pCreateInfo
->mipLodBias
, -16, 15.996),
441 .AnisotropicAlgorithm
= EWAApproximation
,
442 .MinLOD
= anv_clamp_f(pCreateInfo
->minLod
, 0, 14),
443 .MaxLOD
= anv_clamp_f(pCreateInfo
->maxLod
, 0, 14),
444 .ChromaKeyEnable
= 0,
447 .ShadowFunction
= vk_to_gen_shadow_compare_op
[pCreateInfo
->compareOp
],
448 .CubeSurfaceControlMode
= OVERRIDE
,
450 .BorderColorPointer
= border_color_offset
,
453 .LODClampMagnificationMode
= MIPNONE
,
456 .MaximumAnisotropy
= vk_to_gen_max_anisotropy(pCreateInfo
->maxAnisotropy
),
457 .RAddressMinFilterRoundingEnable
= enable_min_filter_addr_rounding
,
458 .RAddressMagFilterRoundingEnable
= enable_mag_filter_addr_rounding
,
459 .VAddressMinFilterRoundingEnable
= enable_min_filter_addr_rounding
,
460 .VAddressMagFilterRoundingEnable
= enable_mag_filter_addr_rounding
,
461 .UAddressMinFilterRoundingEnable
= enable_min_filter_addr_rounding
,
462 .UAddressMagFilterRoundingEnable
= enable_mag_filter_addr_rounding
,
463 .TrilinearFilterQuality
= 0,
464 .NonnormalizedCoordinateEnable
= pCreateInfo
->unnormalizedCoordinates
,
465 .TCXAddressControlMode
= vk_to_gen_tex_address
[pCreateInfo
->addressModeU
],
466 .TCYAddressControlMode
= vk_to_gen_tex_address
[pCreateInfo
->addressModeV
],
467 .TCZAddressControlMode
= vk_to_gen_tex_address
[pCreateInfo
->addressModeW
],
470 .ReductionType
= sampler_reduction_mode
,
471 .ReductionTypeEnable
= enable_sampler_reduction
,
475 GENX(SAMPLER_STATE_pack
)(NULL
, sampler
->state
[p
], &sampler_state
);
477 if (sampler
->bindless_state
.map
) {
478 memcpy(sampler
->bindless_state
.map
+ p
* 32,
479 sampler
->state
[p
], GENX(SAMPLER_STATE_length
) * 4);
483 *pSampler
= anv_sampler_to_handle(sampler
);