projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
swr/rast: constify swr rasterizer
[mesa.git] / src / gallium / drivers / swr / rasterizer / core / backend_impl.h
1 /****************************************************************************
2 * Copyright (C) 2014-2015 Intel Corporation. All Rights Reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * @file backend.h
24 *
25 * @brief Backend handles rasterization, pixel shading and output merger
26 * operations.
27 *
28 ******************************************************************************/
29 #pragma once
30
31 void InitBackendSingleFuncTable(PFN_BACKEND_FUNC(&table)[SWR_INPUT_COVERAGE_COUNT][2][2]);
32 void InitBackendSampleFuncTable(PFN_BACKEND_FUNC(&table)[SWR_MULTISAMPLE_TYPE_COUNT][SWR_INPUT_COVERAGE_COUNT][2][2]);
33
34 static INLINE void CalcSampleBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext);
35
36
37 enum SWR_BACKEND_FUNCS
38 {
39 SWR_BACKEND_SINGLE_SAMPLE,
40 SWR_BACKEND_MSAA_PIXEL_RATE,
41 SWR_BACKEND_MSAA_SAMPLE_RATE,
42 SWR_BACKEND_FUNCS_MAX,
43 };
44
45 #if KNOB_SIMD_WIDTH == 8
46 static const __m256 vCenterOffsetsX = __m256{0.5, 1.5, 0.5, 1.5, 2.5, 3.5, 2.5, 3.5};
47 static const __m256 vCenterOffsetsY = __m256{0.5, 0.5, 1.5, 1.5, 0.5, 0.5, 1.5, 1.5};
48 static const __m256 vULOffsetsX = __m256{0.0, 1.0, 0.0, 1.0, 2.0, 3.0, 2.0, 3.0};
49 static const __m256 vULOffsetsY = __m256{0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0};
50 #define MASK 0xff
51 #endif
52
53 static INLINE simdmask ComputeUserClipMask(uint8_t clipMask, float* pUserClipBuffer, simdscalar const &vI, simdscalar const &vJ)
54 {
55 simdscalar vClipMask = _simd_setzero_ps();
56 uint32_t numClipDistance = _mm_popcnt_u32(clipMask);
57
58 for (uint32_t i = 0; i < numClipDistance; ++i)
59 {
60 // pull triangle clip distance values from clip buffer
61 simdscalar vA = _simd_broadcast_ss(pUserClipBuffer++);
62 simdscalar vB = _simd_broadcast_ss(pUserClipBuffer++);
63 simdscalar vC = _simd_broadcast_ss(pUserClipBuffer++);
64
65 // interpolate
66 simdscalar vInterp = vplaneps(vA, vB, vC, vI, vJ);
67
68 // clip if interpolated clip distance is < 0 || NAN
69 simdscalar vCull = _simd_cmp_ps(_simd_setzero_ps(), vInterp, _CMP_NLE_UQ);
70
71 vClipMask = _simd_or_ps(vClipMask, vCull);
72 }
73
74 return _simd_movemask_ps(vClipMask);
75 }
76
77 INLINE static uint32_t RasterTileColorOffset(uint32_t sampleNum)
78 {
79 static const uint32_t RasterTileColorOffsets[16]
80 { 0,
81 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8),
82 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 2,
83 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 3,
84 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 4,
85 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 5,
86 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 6,
87 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 7,
88 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 8,
89 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 9,
90 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 10,
91 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 11,
92 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 12,
93 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 13,
94 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 14,
95 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp / 8) * 15,
96 };
97 assert(sampleNum < 16);
98 return RasterTileColorOffsets[sampleNum];
99 }
100
101 INLINE static uint32_t RasterTileDepthOffset(uint32_t sampleNum)
102 {
103 static const uint32_t RasterTileDepthOffsets[16]
104 { 0,
105 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8),
106 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 2,
107 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 3,
108 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 4,
109 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 5,
110 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 6,
111 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 7,
112 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 8,
113 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 9,
114 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 10,
115 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 11,
116 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 12,
117 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 13,
118 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 14,
119 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp / 8) * 15,
120 };
121 assert(sampleNum < 16);
122 return RasterTileDepthOffsets[sampleNum];
123 }
124
125 INLINE static uint32_t RasterTileStencilOffset(uint32_t sampleNum)
126 {
127 static const uint32_t RasterTileStencilOffsets[16]
128 { 0,
129 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8),
130 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 2,
131 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 3,
132 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 4,
133 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 5,
134 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 6,
135 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 7,
136 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 8,
137 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 9,
138 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 10,
139 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 11,
140 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 12,
141 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 13,
142 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 14,
143 (KNOB_TILE_X_DIM * KNOB_TILE_Y_DIM * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp / 8) * 15,
144 };
145 assert(sampleNum < 16);
146 return RasterTileStencilOffsets[sampleNum];
147 }
148
149 template<typename T, uint32_t InputCoverage>
150 struct generateInputCoverage
151 {
152 INLINE generateInputCoverage(const uint64_t *const coverageMask, uint32_t (&inputMask)[KNOB_SIMD_WIDTH], const uint32_t sampleMask)
153 {
154 // will need to update for avx512
155 assert(KNOB_SIMD_WIDTH == 8);
156
157 simdscalari mask[2];
158 simdscalari sampleCoverage[2];
159
160 if(T::bIsCenterPattern)
161 {
162 // center coverage is the same for all samples; just broadcast to the sample slots
163 uint32_t centerCoverage = ((uint32_t)(*coverageMask) & MASK);
164 if(T::MultisampleT::numSamples == 1)
165 {
166 sampleCoverage[0] = _simd_set_epi32(0, 0, 0, 0, 0, 0, 0, centerCoverage);
167 }
168 else if(T::MultisampleT::numSamples == 2)
169 {
170 sampleCoverage[0] = _simd_set_epi32(0, 0, 0, 0, 0, 0, centerCoverage, centerCoverage);
171 }
172 else if(T::MultisampleT::numSamples == 4)
173 {
174 sampleCoverage[0] = _simd_set_epi32(0, 0, 0, 0, centerCoverage, centerCoverage, centerCoverage, centerCoverage);
175 }
176 else if(T::MultisampleT::numSamples == 8)
177 {
178 sampleCoverage[0] = _simd_set1_epi32(centerCoverage);
179 }
180 else if(T::MultisampleT::numSamples == 16)
181 {
182 sampleCoverage[0] = _simd_set1_epi32(centerCoverage);
183 sampleCoverage[1] = _simd_set1_epi32(centerCoverage);
184 }
185 }
186 else
187 {
188 simdscalari src = _simd_set1_epi32(0);
189 simdscalari index0 = _simd_set_epi32(7, 6, 5, 4, 3, 2, 1, 0), index1;
190
191 if(T::MultisampleT::numSamples == 1)
192 {
193 mask[0] = _simd_set_epi32(0, 0, 0, 0, 0, 0, 0, -1);
194 }
195 else if(T::MultisampleT::numSamples == 2)
196 {
197 mask[0] = _simd_set_epi32(0, 0, 0, 0, 0, 0, -1, -1);
198 }
199 else if(T::MultisampleT::numSamples == 4)
200 {
201 mask[0] = _simd_set_epi32(0, 0, 0, 0, -1, -1, -1, -1);
202 }
203 else if(T::MultisampleT::numSamples == 8)
204 {
205 mask[0] = _simd_set1_epi32(-1);
206 }
207 else if(T::MultisampleT::numSamples == 16)
208 {
209 mask[0] = _simd_set1_epi32(-1);
210 mask[1] = _simd_set1_epi32(-1);
211 index1 = _simd_set_epi32(15, 14, 13, 12, 11, 10, 9, 8);
212 }
213
214 // gather coverage for samples 0-7
215 sampleCoverage[0] = _mm256_castps_si256(_simd_mask_i32gather_ps(_mm256_castsi256_ps(src), (const float*)coverageMask, index0, _mm256_castsi256_ps(mask[0]), 8));
216 if(T::MultisampleT::numSamples > 8)
217 {
218 // gather coverage for samples 8-15
219 sampleCoverage[1] = _mm256_castps_si256(_simd_mask_i32gather_ps(_mm256_castsi256_ps(src), (const float*)coverageMask, index1, _mm256_castsi256_ps(mask[1]), 8));
220 }
221 }
222
223 mask[0] = _mm256_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0xC, 0x8, 0x4, 0x0,
224 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0xC, 0x8, 0x4, 0x0);
225 // pull out the 8bit 4x2 coverage for samples 0-7 into the lower 32 bits of each 128bit lane
226 simdscalari packedCoverage0 = _simd_shuffle_epi8(sampleCoverage[0], mask[0]);
227
228 simdscalari packedCoverage1;
229 if(T::MultisampleT::numSamples > 8)
230 {
231 // pull out the 8bit 4x2 coverage for samples 8-15 into the lower 32 bits of each 128bit lane
232 packedCoverage1 = _simd_shuffle_epi8(sampleCoverage[1], mask[0]);
233 }
234
235 #if (KNOB_ARCH == KNOB_ARCH_AVX)
236 // pack lower 32 bits of each 128 bit lane into lower 64 bits of single 128 bit lane
237 simdscalari hiToLow = _mm256_permute2f128_si256(packedCoverage0, packedCoverage0, 0x83);
238 simdscalar shufRes = _mm256_shuffle_ps(_mm256_castsi256_ps(hiToLow), _mm256_castsi256_ps(hiToLow), _MM_SHUFFLE(1, 1, 0, 1));
239 packedCoverage0 = _mm256_castps_si256(_mm256_blend_ps(_mm256_castsi256_ps(packedCoverage0), shufRes, 0xFE));
240
241 simdscalari packedSampleCoverage;
242 if(T::MultisampleT::numSamples > 8)
243 {
244 // pack lower 32 bits of each 128 bit lane into upper 64 bits of single 128 bit lane
245 hiToLow = _mm256_permute2f128_si256(packedCoverage1, packedCoverage1, 0x83);
246 shufRes = _mm256_shuffle_ps(_mm256_castsi256_ps(hiToLow), _mm256_castsi256_ps(hiToLow), _MM_SHUFFLE(1, 1, 0, 1));
247 shufRes = _mm256_blend_ps(_mm256_castsi256_ps(packedCoverage1), shufRes, 0xFE);
248 packedCoverage1 = _mm256_castps_si256(_mm256_castpd_ps(_mm256_shuffle_pd(_mm256_castps_pd(shufRes), _mm256_castps_pd(shufRes), 0x01)));
249 packedSampleCoverage = _mm256_castps_si256(_mm256_blend_ps(_mm256_castsi256_ps(packedCoverage0), _mm256_castsi256_ps(packedCoverage1), 0xFC));
250 }
251 else
252 {
253 packedSampleCoverage = packedCoverage0;
254 }
255 #else
256 simdscalari permMask = _simd_set_epi32(0x7, 0x7, 0x7, 0x7, 0x7, 0x7, 0x4, 0x0);
257 // pack lower 32 bits of each 128 bit lane into lower 64 bits of single 128 bit lane
258 packedCoverage0 = _mm256_permutevar8x32_epi32(packedCoverage0, permMask);
259
260 simdscalari packedSampleCoverage;
261 if(T::MultisampleT::numSamples > 8)
262 {
263 permMask = _simd_set_epi32(0x7, 0x7, 0x7, 0x7, 0x4, 0x0, 0x7, 0x7);
264 // pack lower 32 bits of each 128 bit lane into upper 64 bits of single 128 bit lane
265 packedCoverage1 = _mm256_permutevar8x32_epi32(packedCoverage1, permMask);
266
267 // blend coverage masks for samples 0-7 and samples 8-15 into single 128 bit lane
268 packedSampleCoverage = _mm256_blend_epi32(packedCoverage0, packedCoverage1, 0x0C);
269 }
270 else
271 {
272 packedSampleCoverage = packedCoverage0;
273 }
274 #endif
275
276 for(int32_t i = KNOB_SIMD_WIDTH - 1; i >= 0; i--)
277 {
278 // convert packed sample coverage masks into single coverage masks for all samples for each pixel in the 4x2
279 inputMask[i] = _simd_movemask_epi8(packedSampleCoverage);
280
281 if(!T::bForcedSampleCount)
282 {
283 // input coverage has to be anded with sample mask if MSAA isn't forced on
284 inputMask[i] &= sampleMask;
285 }
286
287 // shift to the next pixel in the 4x2
288 packedSampleCoverage = _simd_slli_epi32(packedSampleCoverage, 1);
289 }
290 }
291
292 INLINE generateInputCoverage(const uint64_t *const coverageMask, simdscalar &inputCoverage, const uint32_t sampleMask)
293 {
294 uint32_t inputMask[KNOB_SIMD_WIDTH];
295 generateInputCoverage<T, T::InputCoverage>(coverageMask, inputMask, sampleMask);
296 inputCoverage = _simd_castsi_ps(_simd_set_epi32(inputMask[7], inputMask[6], inputMask[5], inputMask[4], inputMask[3], inputMask[2], inputMask[1], inputMask[0]));
297 }
298
299 };
300
301 template<typename T>
302 struct generateInputCoverage<T, SWR_INPUT_COVERAGE_INNER_CONSERVATIVE>
303 {
304 INLINE generateInputCoverage(const uint64_t *const coverageMask, simdscalar &inputCoverage, const uint32_t sampleMask)
305 {
306 // will need to update for avx512
307 assert(KNOB_SIMD_WIDTH == 8);
308 simdscalari vec = _simd_set1_epi32(coverageMask[0]);
309 const simdscalari bit = _simd_set_epi32(0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01);
310 vec = _simd_and_si(vec, bit);
311 vec = _simd_cmplt_epi32(_simd_setzero_si(), vec);
312 vec = _simd_blendv_epi32(_simd_setzero_si(), _simd_set1_epi32(1), vec);
313 inputCoverage = _simd_castsi_ps(vec);
314 }
315
316 INLINE generateInputCoverage(const uint64_t *const coverageMask, uint32_t (&inputMask)[KNOB_SIMD_WIDTH], const uint32_t sampleMask)
317 {
318 uint32_t simdCoverage = (coverageMask[0] & MASK);
319 static const uint32_t FullCoverageMask = (1 << T::MultisampleT::numSamples) - 1;
320 for(int i = 0; i < KNOB_SIMD_WIDTH; i++)
321 {
322 // set all samples to covered if conservative coverage mask is set for that pixel
323 inputMask[i] = (((1 << i) & simdCoverage) > 0) ? FullCoverageMask : 0;
324 }
325 }
326 };
327
328 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
329 // Centroid behaves exactly as follows :
330 // (1) If all samples in the primitive are covered, the attribute is evaluated at the pixel center (even if the sample pattern does not happen to
331 // have a sample location there).
332 // (2) Else the attribute is evaluated at the first covered sample, in increasing order of sample index, where sample coverage is after ANDing the
333 // coverage with the SampleMask Rasterizer State.
334 // (3) If no samples are covered, such as on helper pixels executed off the bounds of a primitive to fill out 2x2 pixel stamps, the attribute is
335 // evaluated as follows : If the SampleMask Rasterizer state is a subset of the samples in the pixel, then the first sample covered by the
336 // SampleMask Rasterizer State is the evaluation point.Otherwise (full SampleMask), the pixel center is the evaluation point.
337 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
338 template<typename T>
339 INLINE void CalcCentroidPos(SWR_PS_CONTEXT &psContext, const SWR_MULTISAMPLE_POS& samplePos,
340 const uint64_t *const coverageMask, const uint32_t sampleMask,
341 simdscalar const &vXSamplePosUL, simdscalar const &vYSamplePosUL)
342 {
343 uint32_t inputMask[KNOB_SIMD_WIDTH];
344 generateInputCoverage<T, T::InputCoverage>(coverageMask, inputMask, sampleMask);
345
346 // Case (2) - partially covered pixel
347
348 // scan for first covered sample per pixel in the 4x2 span
349 unsigned long sampleNum[KNOB_SIMD_WIDTH];
350 (inputMask[0] > 0) ? (_BitScanForward(&sampleNum[0], inputMask[0])) : (sampleNum[0] = 0);
351 (inputMask[1] > 0) ? (_BitScanForward(&sampleNum[1], inputMask[1])) : (sampleNum[1] = 0);
352 (inputMask[2] > 0) ? (_BitScanForward(&sampleNum[2], inputMask[2])) : (sampleNum[2] = 0);
353 (inputMask[3] > 0) ? (_BitScanForward(&sampleNum[3], inputMask[3])) : (sampleNum[3] = 0);
354 (inputMask[4] > 0) ? (_BitScanForward(&sampleNum[4], inputMask[4])) : (sampleNum[4] = 0);
355 (inputMask[5] > 0) ? (_BitScanForward(&sampleNum[5], inputMask[5])) : (sampleNum[5] = 0);
356 (inputMask[6] > 0) ? (_BitScanForward(&sampleNum[6], inputMask[6])) : (sampleNum[6] = 0);
357 (inputMask[7] > 0) ? (_BitScanForward(&sampleNum[7], inputMask[7])) : (sampleNum[7] = 0);
358
359 // look up and set the sample offsets from UL pixel corner for first covered sample
360 simdscalar vXSample = _simd_set_ps(samplePos.X(sampleNum[7]),
361 samplePos.X(sampleNum[6]),
362 samplePos.X(sampleNum[5]),
363 samplePos.X(sampleNum[4]),
364 samplePos.X(sampleNum[3]),
365 samplePos.X(sampleNum[2]),
366 samplePos.X(sampleNum[1]),
367 samplePos.X(sampleNum[0]));
368
369 simdscalar vYSample = _simd_set_ps(samplePos.Y(sampleNum[7]),
370 samplePos.Y(sampleNum[6]),
371 samplePos.Y(sampleNum[5]),
372 samplePos.Y(sampleNum[4]),
373 samplePos.Y(sampleNum[3]),
374 samplePos.Y(sampleNum[2]),
375 samplePos.Y(sampleNum[1]),
376 samplePos.Y(sampleNum[0]));
377 // add sample offset to UL pixel corner
378 vXSample = _simd_add_ps(vXSamplePosUL, vXSample);
379 vYSample = _simd_add_ps(vYSamplePosUL, vYSample);
380
381 // Case (1) and case (3b) - All samples covered or not covered with full SampleMask
382 static const simdscalari vFullyCoveredMask = T::MultisampleT::FullSampleMask();
383 simdscalari vInputCoveragei = _simd_set_epi32(inputMask[7], inputMask[6], inputMask[5], inputMask[4], inputMask[3], inputMask[2], inputMask[1], inputMask[0]);
384 simdscalari vAllSamplesCovered = _simd_cmpeq_epi32(vInputCoveragei, vFullyCoveredMask);
385
386 static const simdscalari vZero = _simd_setzero_si();
387 const simdscalari vSampleMask = _simd_and_si(_simd_set1_epi32(sampleMask), vFullyCoveredMask);
388 simdscalari vNoSamplesCovered = _simd_cmpeq_epi32(vInputCoveragei, vZero);
389 simdscalari vIsFullSampleMask = _simd_cmpeq_epi32(vSampleMask, vFullyCoveredMask);
390 simdscalari vCase3b = _simd_and_si(vNoSamplesCovered, vIsFullSampleMask);
391
392 simdscalari vEvalAtCenter = _simd_or_si(vAllSamplesCovered, vCase3b);
393
394 // set the centroid position based on results from above
395 psContext.vX.centroid = _simd_blendv_ps(vXSample, psContext.vX.center, _simd_castsi_ps(vEvalAtCenter));
396 psContext.vY.centroid = _simd_blendv_ps(vYSample, psContext.vY.center, _simd_castsi_ps(vEvalAtCenter));
397
398 // Case (3a) No samples covered and partial sample mask
399 simdscalari vSomeSampleMaskSamples = _simd_cmplt_epi32(vSampleMask, vFullyCoveredMask);
400 // sample mask should never be all 0's for this case, but handle it anyways
401 unsigned long firstCoveredSampleMaskSample = 0;
402 (sampleMask > 0) ? (_BitScanForward(&firstCoveredSampleMaskSample, sampleMask)) : (firstCoveredSampleMaskSample = 0);
403
404 simdscalari vCase3a = _simd_and_si(vNoSamplesCovered, vSomeSampleMaskSamples);
405
406 vXSample = _simd_set1_ps(samplePos.X(firstCoveredSampleMaskSample));
407 vYSample = _simd_set1_ps(samplePos.Y(firstCoveredSampleMaskSample));
408
409 // blend in case 3a pixel locations
410 psContext.vX.centroid = _simd_blendv_ps(psContext.vX.centroid, vXSample, _simd_castsi_ps(vCase3a));
411 psContext.vY.centroid = _simd_blendv_ps(psContext.vY.centroid, vYSample, _simd_castsi_ps(vCase3a));
412 }
413
414 INLINE void CalcCentroidBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext,
415 const simdscalar &vXSamplePosUL, const simdscalar &vYSamplePosUL)
416 {
417 // evaluate I,J
418 psContext.vI.centroid = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.centroid, psContext.vY.centroid);
419 psContext.vJ.centroid = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.centroid, psContext.vY.centroid);
420 psContext.vI.centroid = _simd_mul_ps(psContext.vI.centroid, coeffs.vRecipDet);
421 psContext.vJ.centroid = _simd_mul_ps(psContext.vJ.centroid, coeffs.vRecipDet);
422
423 // interpolate 1/w
424 psContext.vOneOverW.centroid = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.centroid, psContext.vJ.centroid);
425 }
426
427 INLINE simdmask CalcDepthBoundsAcceptMask(simdscalar const &z, float minz, float maxz)
428 {
429 const simdscalar minzMask = _simd_cmpge_ps(z, _simd_set1_ps(minz));
430 const simdscalar maxzMask = _simd_cmple_ps(z, _simd_set1_ps(maxz));
431
432 return _simd_movemask_ps(_simd_and_ps(minzMask, maxzMask));
433 }
434
435 template<typename T>
436 INLINE uint32_t GetNumOMSamples(SWR_MULTISAMPLE_COUNT blendSampleCount)
437 {
438 // RT has to be single sample if we're in forcedMSAA mode
439 if(T::bForcedSampleCount && (T::MultisampleT::sampleCount > SWR_MULTISAMPLE_1X))
440 {
441 return 1;
442 }
443 // unless we're forced to single sample, in which case we run the OM at the sample count of the RT
444 else if(T::bForcedSampleCount && (T::MultisampleT::sampleCount == SWR_MULTISAMPLE_1X))
445 {
446 return GetNumSamples(blendSampleCount);
447 }
448 // else we're in normal MSAA mode and rasterizer and OM are running at the same sample count
449 else
450 {
451 return T::MultisampleT::numSamples;
452 }
453 }
454
455 inline void SetupBarycentricCoeffs(BarycentricCoeffs *coeffs, const SWR_TRIANGLE_DESC &work)
456 {
457 // broadcast scalars
458
459 coeffs->vIa = _simd_broadcast_ss(&work.I[0]);
460 coeffs->vIb = _simd_broadcast_ss(&work.I[1]);
461 coeffs->vIc = _simd_broadcast_ss(&work.I[2]);
462
463 coeffs->vJa = _simd_broadcast_ss(&work.J[0]);
464 coeffs->vJb = _simd_broadcast_ss(&work.J[1]);
465 coeffs->vJc = _simd_broadcast_ss(&work.J[2]);
466
467 coeffs->vZa = _simd_broadcast_ss(&work.Z[0]);
468 coeffs->vZb = _simd_broadcast_ss(&work.Z[1]);
469 coeffs->vZc = _simd_broadcast_ss(&work.Z[2]);
470
471 coeffs->vRecipDet = _simd_broadcast_ss(&work.recipDet);
472
473 coeffs->vAOneOverW = _simd_broadcast_ss(&work.OneOverW[0]);
474 coeffs->vBOneOverW = _simd_broadcast_ss(&work.OneOverW[1]);
475 coeffs->vCOneOverW = _simd_broadcast_ss(&work.OneOverW[2]);
476 }
477
478 inline void SetupRenderBuffers(uint8_t *pColorBuffer[SWR_NUM_RENDERTARGETS], uint8_t **pDepthBuffer, uint8_t **pStencilBuffer, uint32_t colorHotTileMask, RenderOutputBuffers &renderBuffers)
479 {
480
481 DWORD index;
482 while (_BitScanForward(&index, colorHotTileMask))
483 {
484 assert(index < SWR_NUM_RENDERTARGETS);
485 colorHotTileMask &= ~(1 << index);
486 pColorBuffer[index] = renderBuffers.pColor[index];
487 }
488
489 if (pDepthBuffer)
490 {
491 *pDepthBuffer = renderBuffers.pDepth;
492 }
493
494 if (pStencilBuffer)
495 {
496 *pStencilBuffer = renderBuffers.pStencil;;
497 }
498 }
499
500 template<typename T>
501 void SetupPixelShaderContext(SWR_PS_CONTEXT *psContext, const SWR_MULTISAMPLE_POS& samplePos, SWR_TRIANGLE_DESC &work)
502 {
503 psContext->pAttribs = work.pAttribs;
504 psContext->pPerspAttribs = work.pPerspAttribs;
505 psContext->frontFace = work.triFlags.frontFacing;
506 psContext->renderTargetArrayIndex = work.triFlags.renderTargetArrayIndex;
507
508 // save Ia/Ib/Ic and Ja/Jb/Jc if we need to reevaluate i/j/k in the shader because of pull attribs
509 psContext->I = work.I;
510 psContext->J = work.J;
511
512 psContext->recipDet = work.recipDet;
513 psContext->pRecipW = work.pRecipW;
514 psContext->pSamplePosX = samplePos.X();//reinterpret_cast<const float *>(&T::MultisampleT::samplePosX);
515 psContext->pSamplePosY = samplePos.Y();//reinterpret_cast<const float *>(&T::MultisampleT::samplePosY);
516 psContext->rasterizerSampleCount = T::MultisampleT::numSamples;
517 psContext->sampleIndex = 0;
518 }
519
520 template<typename T, bool IsSingleSample>
521 void CalcCentroid(SWR_PS_CONTEXT *psContext, const SWR_MULTISAMPLE_POS& samplePos,
522 const BarycentricCoeffs &coeffs, const uint64_t * const coverageMask, uint32_t sampleMask)
523 {
524 if (IsSingleSample) // if (T::MultisampleT::numSamples == 1) // doesn't cut it, the centroid positions are still different
525 {
526 // for 1x case, centroid is pixel center
527 psContext->vX.centroid = psContext->vX.center;
528 psContext->vY.centroid = psContext->vY.center;
529 psContext->vI.centroid = psContext->vI.center;
530 psContext->vJ.centroid = psContext->vJ.center;
531 psContext->vOneOverW.centroid = psContext->vOneOverW.center;
532 }
533 else
534 {
535 if (T::bCentroidPos)
536 {
537 ///@ todo: don't need to genererate input coverage 2x if input coverage and centroid
538 if (T::bIsCenterPattern)
539 {
540 psContext->vX.centroid = _simd_add_ps(psContext->vX.UL, _simd_set1_ps(0.5f));
541 psContext->vY.centroid = _simd_add_ps(psContext->vY.UL, _simd_set1_ps(0.5f));
542 }
543 else
544 {
545 // add param: const uint32_t inputMask[KNOB_SIMD_WIDTH] to eliminate 'generate coverage 2X'..
546 CalcCentroidPos<T>(*psContext, samplePos, coverageMask, sampleMask, psContext->vX.UL, psContext->vY.UL);
547 }
548
549 CalcCentroidBarycentrics(coeffs, *psContext, psContext->vX.UL, psContext->vY.UL);
550 }
551 else
552 {
553 psContext->vX.centroid = psContext->vX.sample;
554 psContext->vY.centroid = psContext->vY.sample;
555 }
556 }
557 }
558
559 template<typename T>
560 struct PixelRateZTestLoop
561 {
562 PixelRateZTestLoop(DRAW_CONTEXT *DC, uint32_t _workerId, const SWR_TRIANGLE_DESC &Work, const BarycentricCoeffs& Coeffs, const API_STATE& apiState,
563 uint8_t*& depthBuffer, uint8_t*& stencilBuffer, const uint8_t ClipDistanceMask) :
564 pDC(DC), workerId(_workerId), work(Work), coeffs(Coeffs), state(apiState), psState(apiState.psState),
565 samplePos(state.rastState.samplePositions),
566 clipDistanceMask(ClipDistanceMask), pDepthBuffer(depthBuffer), pStencilBuffer(stencilBuffer){};
567
568 INLINE
569 uint32_t operator()(simdscalar& activeLanes, SWR_PS_CONTEXT& psContext,
570 const CORE_BUCKETS BEDepthBucket, uint32_t currentSimdIn8x8 = 0)
571 {
572 SWR_CONTEXT *pContext = pDC->pContext;
573
574 uint32_t statCount = 0;
575 simdscalar anyDepthSamplePassed = _simd_setzero_ps();
576 for(uint32_t sample = 0; sample < T::MultisampleT::numCoverageSamples; sample++)
577 {
578 const uint8_t *pCoverageMask = (uint8_t*)&work.coverageMask[sample];
579 vCoverageMask[sample] = _simd_and_ps(activeLanes, _simd_vmask_ps(pCoverageMask[currentSimdIn8x8] & MASK));
580
581 if(!_simd_movemask_ps(vCoverageMask[sample]))
582 {
583 vCoverageMask[sample] = depthPassMask[sample] = stencilPassMask[sample] = _simd_setzero_ps();
584 continue;
585 }
586
587 // offset depth/stencil buffers current sample
588 uint8_t *pDepthSample = pDepthBuffer + RasterTileDepthOffset(sample);
589 uint8_t * pStencilSample = pStencilBuffer + RasterTileStencilOffset(sample);
590
591 if (state.depthHottileEnable && state.depthBoundsState.depthBoundsTestEnable)
592 {
593 static_assert(KNOB_DEPTH_HOT_TILE_FORMAT == R32_FLOAT, "Unsupported depth hot tile format");
594
595 const simdscalar z = _simd_load_ps(reinterpret_cast<const float *>(pDepthSample));
596
597 const float minz = state.depthBoundsState.depthBoundsTestMinValue;
598 const float maxz = state.depthBoundsState.depthBoundsTestMaxValue;
599
600 vCoverageMask[sample] = _simd_and_ps(vCoverageMask[sample], _simd_vmask_ps(CalcDepthBoundsAcceptMask(z, minz, maxz)));
601 }
602
603 AR_BEGIN(BEBarycentric, pDC->drawId);
604
605 // calculate per sample positions
606 psContext.vX.sample = _simd_add_ps(psContext.vX.UL, samplePos.vX(sample));
607 psContext.vY.sample = _simd_add_ps(psContext.vY.UL, samplePos.vY(sample));
608
609 // calc I & J per sample
610 CalcSampleBarycentrics(coeffs, psContext);
611
612 if(psState.writesODepth)
613 {
614 {
615 // broadcast and test oDepth(psContext.vZ) written from the PS for each sample
616 vZ[sample] = psContext.vZ;
617 }
618 }
619 else
620 {
621 vZ[sample] = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.sample, psContext.vJ.sample);
622 vZ[sample] = state.pfnQuantizeDepth(vZ[sample]);
623 }
624
625 AR_END(BEBarycentric, 0);
626
627 ///@todo: perspective correct vs non-perspective correct clipping?
628 // if clip distances are enabled, we need to interpolate for each sample
629 if(clipDistanceMask)
630 {
631 uint8_t clipMask = ComputeUserClipMask(clipDistanceMask, work.pUserClipBuffer, psContext.vI.sample, psContext.vJ.sample);
632
633 vCoverageMask[sample] = _simd_and_ps(vCoverageMask[sample], _simd_vmask_ps(~clipMask));
634 }
635
636 // ZTest for this sample
637 ///@todo Need to uncomment out this bucket.
638 //AR_BEGIN(BEDepthBucket, pDC->drawId);
639 depthPassMask[sample] = vCoverageMask[sample];
640 stencilPassMask[sample] = vCoverageMask[sample];
641 depthPassMask[sample] = DepthStencilTest(&state, work.triFlags.frontFacing, work.triFlags.viewportIndex,
642 vZ[sample], pDepthSample, vCoverageMask[sample],
643 pStencilSample, &stencilPassMask[sample]);
644 //AR_END(BEDepthBucket, 0);
645
646 // early-exit if no pixels passed depth or earlyZ is forced on
647 if(psState.forceEarlyZ || !_simd_movemask_ps(depthPassMask[sample]))
648 {
649 DepthStencilWrite(&state.vp[work.triFlags.viewportIndex], &state.depthStencilState, work.triFlags.frontFacing, vZ[sample],
650 pDepthSample, depthPassMask[sample], vCoverageMask[sample], pStencilSample, stencilPassMask[sample]);
651
652 if(!_simd_movemask_ps(depthPassMask[sample]))
653 {
654 continue;
655 }
656 }
657 anyDepthSamplePassed = _simd_or_ps(anyDepthSamplePassed, depthPassMask[sample]);
658 uint32_t statMask = _simd_movemask_ps(depthPassMask[sample]);
659 statCount += _mm_popcnt_u32(statMask);
660 }
661
662 activeLanes = _simd_and_ps(anyDepthSamplePassed, activeLanes);
663 // return number of samples that passed depth and coverage
664 return statCount;
665 }
666
667 // saved depth/stencil/coverage masks and interpolated Z used in OM and DepthWrite
668 simdscalar vZ[T::MultisampleT::numCoverageSamples];
669 simdscalar vCoverageMask[T::MultisampleT::numCoverageSamples];
670 simdscalar depthPassMask[T::MultisampleT::numCoverageSamples];
671 simdscalar stencilPassMask[T::MultisampleT::numCoverageSamples];
672
673 private:
674 // functor inputs
675 DRAW_CONTEXT* pDC;
676 uint32_t workerId;
677
678 const SWR_TRIANGLE_DESC& work;
679 const BarycentricCoeffs& coeffs;
680 const API_STATE& state;
681 const SWR_PS_STATE& psState;
682 const SWR_MULTISAMPLE_POS& samplePos;
683 const uint8_t clipDistanceMask;
684 uint8_t*& pDepthBuffer;
685 uint8_t*& pStencilBuffer;
686 };
687
688 INLINE void CalcPixelBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext)
689 {
690 // evaluate I,J
691 psContext.vI.center = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.center, psContext.vY.center);
692 psContext.vJ.center = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.center, psContext.vY.center);
693 psContext.vI.center = _simd_mul_ps(psContext.vI.center, coeffs.vRecipDet);
694 psContext.vJ.center = _simd_mul_ps(psContext.vJ.center, coeffs.vRecipDet);
695
696 // interpolate 1/w
697 psContext.vOneOverW.center = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.center, psContext.vJ.center);
698 }
699
700 static INLINE void CalcSampleBarycentrics(const BarycentricCoeffs& coeffs, SWR_PS_CONTEXT &psContext)
701 {
702 // evaluate I,J
703 psContext.vI.sample = vplaneps(coeffs.vIa, coeffs.vIb, coeffs.vIc, psContext.vX.sample, psContext.vY.sample);
704 psContext.vJ.sample = vplaneps(coeffs.vJa, coeffs.vJb, coeffs.vJc, psContext.vX.sample, psContext.vY.sample);
705 psContext.vI.sample = _simd_mul_ps(psContext.vI.sample, coeffs.vRecipDet);
706 psContext.vJ.sample = _simd_mul_ps(psContext.vJ.sample, coeffs.vRecipDet);
707
708 // interpolate 1/w
709 psContext.vOneOverW.sample = vplaneps(coeffs.vAOneOverW, coeffs.vBOneOverW, coeffs.vCOneOverW, psContext.vI.sample, psContext.vJ.sample);
710 }
711
712 // Merge Output to 4x2 SIMD Tile Format
713 INLINE void OutputMerger4x2(SWR_PS_CONTEXT &psContext, uint8_t* (&pColorBase)[SWR_NUM_RENDERTARGETS], uint32_t sample, const SWR_BLEND_STATE *pBlendState,
714 const PFN_BLEND_JIT_FUNC (&pfnBlendFunc)[SWR_NUM_RENDERTARGETS], simdscalar &coverageMask, simdscalar const &depthPassMask, uint32_t renderTargetMask)
715 {
716 // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
717 const uint32_t rasterTileColorOffset = RasterTileColorOffset(sample);
718 simdvector blendOut;
719
720 DWORD rt = 0;
721 while (_BitScanForward(&rt, renderTargetMask))
722 {
723 renderTargetMask &= ~(1 << rt);
724 uint8_t *pColorSample = pColorBase[rt] + rasterTileColorOffset;
725
726 const SWR_RENDER_TARGET_BLEND_STATE *pRTBlend = &pBlendState->renderTarget[rt];
727
728 {
729 // pfnBlendFunc may not update all channels. Initialize with PS output.
730 /// TODO: move this into the blend JIT.
731 blendOut = psContext.shaded[rt];
732
733 // Blend outputs and update coverage mask for alpha test
734 if(pfnBlendFunc[rt] != nullptr)
735 {
736 pfnBlendFunc[rt](
737 pBlendState,
738 psContext.shaded[rt],
739 psContext.shaded[1],
740 psContext.shaded[0].w,
741 sample,
742 pColorSample,
743 blendOut,
744 &psContext.oMask,
745 (simdscalari*)&coverageMask);
746 }
747 }
748
749 // final write mask
750 simdscalari outputMask = _simd_castps_si(_simd_and_ps(coverageMask, depthPassMask));
751
752 ///@todo can only use maskstore fast path if bpc is 32. Assuming hot tile is RGBA32_FLOAT.
753 static_assert(KNOB_COLOR_HOT_TILE_FORMAT == R32G32B32A32_FLOAT, "Unsupported hot tile format");
754
755 const uint32_t simd = KNOB_SIMD_WIDTH * sizeof(float);
756
757 // store with color mask
758 if(!pRTBlend->writeDisableRed)
759 {
760 _simd_maskstore_ps((float*)pColorSample, outputMask, blendOut.x);
761 }
762 if(!pRTBlend->writeDisableGreen)
763 {
764 _simd_maskstore_ps((float*)(pColorSample + simd), outputMask, blendOut.y);
765 }
766 if(!pRTBlend->writeDisableBlue)
767 {
768 _simd_maskstore_ps((float*)(pColorSample + simd * 2), outputMask, blendOut.z);
769 }
770 if(!pRTBlend->writeDisableAlpha)
771 {
772 _simd_maskstore_ps((float*)(pColorSample + simd * 3), outputMask, blendOut.w);
773 }
774 }
775 }
776
777 #if USE_8x2_TILE_BACKEND
778 // Merge Output to 8x2 SIMD16 Tile Format
779 INLINE void OutputMerger8x2(SWR_PS_CONTEXT &psContext, uint8_t* (&pColorBase)[SWR_NUM_RENDERTARGETS], uint32_t sample, const SWR_BLEND_STATE *pBlendState,
780 const PFN_BLEND_JIT_FUNC(&pfnBlendFunc)[SWR_NUM_RENDERTARGETS], simdscalar &coverageMask, simdscalar const &depthPassMask, uint32_t renderTargetMask, bool useAlternateOffset)
781 {
782 // type safety guaranteed from template instantiation in BEChooser<>::GetFunc
783 uint32_t rasterTileColorOffset = RasterTileColorOffset(sample);
784
785 if (useAlternateOffset)
786 {
787 rasterTileColorOffset += sizeof(simdscalar);
788 }
789
790 simdvector blendSrc;
791 simdvector blendOut;
792
793 DWORD rt;
794 while (_BitScanForward(&rt, renderTargetMask))
795 {
796 renderTargetMask &= ~(1 << rt);
797
798 const SWR_RENDER_TARGET_BLEND_STATE *pRTBlend = &pBlendState->renderTarget[rt];
799
800 simdscalar* pColorSample;
801 bool hotTileEnable = !pRTBlend->writeDisableAlpha || !pRTBlend->writeDisableRed || !pRTBlend->writeDisableGreen || !pRTBlend->writeDisableBlue;
802 if (hotTileEnable)
803 {
804 pColorSample = reinterpret_cast<simdscalar *>(pColorBase[rt] + rasterTileColorOffset);
805 blendSrc[0] = pColorSample[0];
806 blendSrc[1] = pColorSample[2];
807 blendSrc[2] = pColorSample[4];
808 blendSrc[3] = pColorSample[6];
809 }
810 else
811 {
812 pColorSample = nullptr;
813 }
814
815 {
816 // pfnBlendFunc may not update all channels. Initialize with PS output.
817 /// TODO: move this into the blend JIT.
818 blendOut = psContext.shaded[rt];
819
820 // Blend outputs and update coverage mask for alpha test
821 if(pfnBlendFunc[rt] != nullptr)
822 {
823 pfnBlendFunc[rt](
824 pBlendState,
825 psContext.shaded[rt],
826 psContext.shaded[1],
827 psContext.shaded[0].w,
828 sample,
829 reinterpret_cast<uint8_t *>(&blendSrc),
830 blendOut,
831 &psContext.oMask,
832 reinterpret_cast<simdscalari *>(&coverageMask));
833 }
834 }
835
836 // final write mask
837 simdscalari outputMask = _simd_castps_si(_simd_and_ps(coverageMask, depthPassMask));
838
839 ///@todo can only use maskstore fast path if bpc is 32. Assuming hot tile is RGBA32_FLOAT.
840 static_assert(KNOB_COLOR_HOT_TILE_FORMAT == R32G32B32A32_FLOAT, "Unsupported hot tile format");
841
842 // store with color mask
843 if (!pRTBlend->writeDisableRed)
844 {
845 _simd_maskstore_ps(reinterpret_cast<float *>(&pColorSample[0]), outputMask, blendOut.x);
846 }
847 if (!pRTBlend->writeDisableGreen)
848 {
849 _simd_maskstore_ps(reinterpret_cast<float *>(&pColorSample[2]), outputMask, blendOut.y);
850 }
851 if (!pRTBlend->writeDisableBlue)
852 {
853 _simd_maskstore_ps(reinterpret_cast<float *>(&pColorSample[4]), outputMask, blendOut.z);
854 }
855 if (!pRTBlend->writeDisableAlpha)
856 {
857 _simd_maskstore_ps(reinterpret_cast<float *>(&pColorSample[6]), outputMask, blendOut.w);
858 }
859 }
860 }
861
862 #endif
863
864 template<typename T>
865 void BackendPixelRate(DRAW_CONTEXT *pDC, uint32_t workerId, uint32_t x, uint32_t y, SWR_TRIANGLE_DESC &work, RenderOutputBuffers &renderBuffers)
866 {
867 ///@todo: Need to move locals off stack to prevent __chkstk's from being generated for the backend
868
869
870 SWR_CONTEXT *pContext = pDC->pContext;
871
872 AR_BEGIN(BEPixelRateBackend, pDC->drawId);
873 AR_BEGIN(BESetup, pDC->drawId);
874
875 const API_STATE &state = GetApiState(pDC);
876
877 BarycentricCoeffs coeffs;
878 SetupBarycentricCoeffs(&coeffs, work);
879
880 SWR_PS_CONTEXT psContext;
881 const SWR_MULTISAMPLE_POS& samplePos = state.rastState.samplePositions;
882 SetupPixelShaderContext<T>(&psContext, samplePos, work);
883
884 uint8_t *pDepthBuffer, *pStencilBuffer;
885 SetupRenderBuffers(psContext.pColorBuffer, &pDepthBuffer, &pStencilBuffer, state.colorHottileEnable, renderBuffers);
886
887 AR_END(BESetup, 0);
888
889 PixelRateZTestLoop<T> PixelRateZTest(pDC, workerId, work, coeffs, state, pDepthBuffer, pStencilBuffer, state.rastState.clipDistanceMask);
890
891 psContext.vY.UL = _simd_add_ps(vULOffsetsY, _simd_set1_ps(static_cast<float>(y)));
892 psContext.vY.center = _simd_add_ps(vCenterOffsetsY, _simd_set1_ps(static_cast<float>(y)));
893
894 const simdscalar dy = _simd_set1_ps(static_cast<float>(SIMD_TILE_Y_DIM));
895
896 for(uint32_t yy = y; yy < y + KNOB_TILE_Y_DIM; yy += SIMD_TILE_Y_DIM)
897 {
898 psContext.vX.UL = _simd_add_ps(vULOffsetsX, _simd_set1_ps(static_cast<float>(x)));
899 psContext.vX.center = _simd_add_ps(vCenterOffsetsX, _simd_set1_ps(static_cast<float>(x)));
900
901 const simdscalar dx = _simd_set1_ps(static_cast<float>(SIMD_TILE_X_DIM));
902
903 for(uint32_t xx = x; xx < x + KNOB_TILE_X_DIM; xx += SIMD_TILE_X_DIM)
904 {
905 #if USE_8x2_TILE_BACKEND
906 const bool useAlternateOffset = ((xx & SIMD_TILE_X_DIM) != 0);
907 #endif
908 simdscalar activeLanes;
909 if(!(work.anyCoveredSamples & MASK)) {goto Endtile;};
910 activeLanes = _simd_vmask_ps(work.anyCoveredSamples & MASK);
911
912 if (T::InputCoverage != SWR_INPUT_COVERAGE_NONE)
913 {
914 const uint64_t* pCoverageMask = (T::InputCoverage == SWR_INPUT_COVERAGE_INNER_CONSERVATIVE) ? &work.innerCoverageMask : &work.coverageMask[0];
915
916 generateInputCoverage<T, T::InputCoverage>(pCoverageMask, psContext.inputMask, state.blendState.sampleMask);
917 }
918
919 AR_BEGIN(BEBarycentric, pDC->drawId);
920
921 CalcPixelBarycentrics(coeffs, psContext);
922
923 CalcCentroid<T, false>(&psContext, samplePos, coeffs, work.coverageMask, state.blendState.sampleMask);
924
925 AR_END(BEBarycentric, 0);
926
927 if(T::bForcedSampleCount)
928 {
929 // candidate pixels (that passed coverage) will cause shader invocation if any bits in the samplemask are set
930 const simdscalar vSampleMask = _simd_castsi_ps(_simd_cmpgt_epi32(_simd_set1_epi32(state.blendState.sampleMask), _simd_setzero_si()));
931 activeLanes = _simd_and_ps(activeLanes, vSampleMask);
932 }
933
934 // Early-Z?
935 if(T::bCanEarlyZ && !T::bForcedSampleCount)
936 {
937 uint32_t depthPassCount = PixelRateZTest(activeLanes, psContext, BEEarlyDepthTest);
938 UPDATE_STAT_BE(DepthPassCount, depthPassCount);
939 AR_EVENT(EarlyDepthInfoPixelRate(depthPassCount, _simd_movemask_ps(activeLanes)));
940 }
941
942 // if we have no covered samples that passed depth at this point, go to next tile
943 if(!_simd_movemask_ps(activeLanes)) { goto Endtile; };
944
945 if(state.psState.usesSourceDepth)
946 {
947 AR_BEGIN(BEBarycentric, pDC->drawId);
948 // interpolate and quantize z
949 psContext.vZ = vplaneps(coeffs.vZa, coeffs.vZb, coeffs.vZc, psContext.vI.center, psContext.vJ.center);
950 psContext.vZ = state.pfnQuantizeDepth(psContext.vZ);
951 AR_END(BEBarycentric, 0);
952 }
953
954 // pixels that are currently active
955 psContext.activeMask = _simd_castps_si(activeLanes);
956 psContext.oMask = T::MultisampleT::FullSampleMask();
957
958 // execute pixel shader
959 AR_BEGIN(BEPixelShader, pDC->drawId);
960 state.psState.pfnPixelShader(GetPrivateState(pDC), &psContext);
961 UPDATE_STAT_BE(PsInvocations, _mm_popcnt_u32(_simd_movemask_ps(activeLanes)));
962 AR_END(BEPixelShader, 0);
963
964 // update active lanes to remove any discarded or oMask'd pixels
965 activeLanes = _simd_castsi_ps(_simd_and_si(psContext.activeMask, _simd_cmpgt_epi32(psContext.oMask, _simd_setzero_si())));
966 if(!_simd_movemask_ps(activeLanes)) { goto Endtile; };
967
968 // late-Z
969 if(!T::bCanEarlyZ && !T::bForcedSampleCount)
970 {
971 uint32_t depthPassCount = PixelRateZTest(activeLanes, psContext, BELateDepthTest);
972 UPDATE_STAT_BE(DepthPassCount, depthPassCount);
973 AR_EVENT(LateDepthInfoPixelRate(depthPassCount, _simd_movemask_ps(activeLanes)));
974 }
975
976 // if we have no covered samples that passed depth at this point, skip OM and go to next tile
977 if(!_simd_movemask_ps(activeLanes)) { goto Endtile; };
978
979 // output merger
980 // loop over all samples, broadcasting the results of the PS to all passing pixels
981 for(uint32_t sample = 0; sample < GetNumOMSamples<T>(state.blendState.sampleCount); sample++)
982 {
983 AR_BEGIN(BEOutputMerger, pDC->drawId);
984 // center pattern does a single coverage/depth/stencil test, standard pattern tests all samples
985 uint32_t coverageSampleNum = (T::bIsCenterPattern) ? 0 : sample;
986 simdscalar coverageMask, depthMask;
987 if(T::bForcedSampleCount)
988 {
989 coverageMask = depthMask = activeLanes;
990 }
991 else
992 {
993 coverageMask = PixelRateZTest.vCoverageMask[coverageSampleNum];
994 depthMask = PixelRateZTest.depthPassMask[coverageSampleNum];
995 if(!_simd_movemask_ps(depthMask))
996 {
997 // stencil should already have been written in early/lateZ tests
998 AR_END(BEOutputMerger, 0);
999 continue;
1000 }
1001 }
1002
1003 // broadcast the results of the PS to all passing pixels
1004 #if USE_8x2_TILE_BACKEND
1005 OutputMerger8x2(psContext, psContext.pColorBuffer, sample, &state.blendState,state.pfnBlendFunc, coverageMask, depthMask, state.psState.renderTargetMask, useAlternateOffset);
1006 #else // USE_8x2_TILE_BACKEND
1007 OutputMerger4x2(psContext, psContext.pColorBuffer, sample, &state.blendState, state.pfnBlendFunc, coverageMask, depthMask, state.psState.renderTargetMask);
1008 #endif // USE_8x2_TILE_BACKEND
1009
1010 if(!state.psState.forceEarlyZ && !T::bForcedSampleCount)
1011 {
1012 uint8_t *pDepthSample = pDepthBuffer + RasterTileDepthOffset(sample);
1013 uint8_t * pStencilSample = pStencilBuffer + RasterTileStencilOffset(sample);
1014
1015 DepthStencilWrite(&state.vp[work.triFlags.viewportIndex], &state.depthStencilState, work.triFlags.frontFacing, PixelRateZTest.vZ[coverageSampleNum],
1016 pDepthSample, depthMask, coverageMask, pStencilSample, PixelRateZTest.stencilPassMask[coverageSampleNum]);
1017 }
1018 AR_END(BEOutputMerger, 0);
1019 }
1020 Endtile:
1021 AR_BEGIN(BEEndTile, pDC->drawId);
1022
1023 for(uint32_t sample = 0; sample < T::MultisampleT::numCoverageSamples; sample++)
1024 {
1025 work.coverageMask[sample] >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1026 }
1027
1028 if(T::InputCoverage == SWR_INPUT_COVERAGE_INNER_CONSERVATIVE)
1029 {
1030 work.innerCoverageMask >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1031 }
1032 work.anyCoveredSamples >>= (SIMD_TILE_Y_DIM * SIMD_TILE_X_DIM);
1033
1034 #if USE_8x2_TILE_BACKEND
1035 if (useAlternateOffset)
1036 {
1037 DWORD rt;
1038 uint32_t rtMask = state.colorHottileEnable;
1039 while (_BitScanForward(&rt, rtMask))
1040 {
1041 rtMask &= ~(1 << rt);
1042 psContext.pColorBuffer[rt] += (2 * KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
1043 }
1044 }
1045 #else
1046 DWORD rt;
1047 uint32_t rtMask = state.colorHottileEnable;
1048 while (_BitScanForward(&rt, rtMask))
1049 {
1050 rtMask &= ~(1 << rt);
1051 psContext.pColorBuffer[rt] += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_COLOR_HOT_TILE_FORMAT>::bpp) / 8;
1052 }
1053 #endif
1054 pDepthBuffer += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_DEPTH_HOT_TILE_FORMAT>::bpp) / 8;
1055 pStencilBuffer += (KNOB_SIMD_WIDTH * FormatTraits<KNOB_STENCIL_HOT_TILE_FORMAT>::bpp) / 8;
1056
1057 AR_END(BEEndTile, 0);
1058
1059 psContext.vX.UL = _simd_add_ps(psContext.vX.UL, dx);
1060 psContext.vX.center = _simd_add_ps(psContext.vX.center, dx);
1061 }
1062
1063 psContext.vY.UL = _simd_add_ps(psContext.vY.UL, dy);
1064 psContext.vY.center = _simd_add_ps(psContext.vY.center, dy);
1065 }
1066
1067 AR_END(BEPixelRateBackend, 0);
1068 }
1069
1070 template<uint32_t sampleCountT = SWR_MULTISAMPLE_1X, uint32_t isCenter = 0,
1071 uint32_t coverage = 0, uint32_t centroid = 0, uint32_t forced = 0, uint32_t canEarlyZ = 0
1072 >
1073 struct SwrBackendTraits
1074 {
1075 static const bool bIsCenterPattern = (isCenter == 1);
1076 static const uint32_t InputCoverage = coverage;
1077 static const bool bCentroidPos = (centroid == 1);
1078 static const bool bForcedSampleCount = (forced == 1);
1079 static const bool bCanEarlyZ = (canEarlyZ == 1);
1080 typedef MultisampleTraits<(SWR_MULTISAMPLE_COUNT)sampleCountT, bIsCenterPattern> MultisampleT;
1081 };