projects / mesa.git / blob
? search:


5ef45caecc352c114e235238298ad629669214e2
[mesa.git] / src / gallium / drivers / swr / swr_shader.cpp
1 /****************************************************************************
2 * Copyright (C) 2015 Intel Corporation. All Rights Reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 ***************************************************************************/
23
24 // llvm redefines DEBUG
25 #pragma push_macro("DEBUG")
26 #undef DEBUG
27 #include "JitManager.h"
28 #include "llvm-c/Core.h"
29 #include "llvm/Support/CBindingWrapping.h"
30 #include "llvm/IR/LegacyPassManager.h"
31 #pragma pop_macro("DEBUG")
32
33 #include "state.h"
34 #include "gen_state_llvm.h"
35 #include "builder.h"
36 #include "functionpasses/passes.h"
37
38 #include "tgsi/tgsi_strings.h"
39 #include "util/u_format.h"
40 #include "util/u_prim.h"
41 #include "gallivm/lp_bld_init.h"
42 #include "gallivm/lp_bld_flow.h"
43 #include "gallivm/lp_bld_struct.h"
44 #include "gallivm/lp_bld_tgsi.h"
45
46 #include "swr_context.h"
47 #include "gen_surf_state_llvm.h"
48 #include "gen_swr_context_llvm.h"
49 #include "swr_resource.h"
50 #include "swr_state.h"
51 #include "swr_screen.h"
52
53 using namespace SwrJit;
54 using namespace llvm;
55
56 static unsigned
57 locate_linkage(ubyte name, ubyte index, struct tgsi_shader_info *info);
58
59 bool operator==(const swr_jit_fs_key &lhs, const swr_jit_fs_key &rhs)
60 {
61 return !memcmp(&lhs, &rhs, sizeof(lhs));
62 }
63
64 bool operator==(const swr_jit_vs_key &lhs, const swr_jit_vs_key &rhs)
65 {
66 return !memcmp(&lhs, &rhs, sizeof(lhs));
67 }
68
69 bool operator==(const swr_jit_fetch_key &lhs, const swr_jit_fetch_key &rhs)
70 {
71 return !memcmp(&lhs, &rhs, sizeof(lhs));
72 }
73
74 bool operator==(const swr_jit_gs_key &lhs, const swr_jit_gs_key &rhs)
75 {
76 return !memcmp(&lhs, &rhs, sizeof(lhs));
77 }
78
79 static void
80 swr_generate_sampler_key(const struct lp_tgsi_info &info,
81 struct swr_context *ctx,
82 enum pipe_shader_type shader_type,
83 struct swr_jit_sampler_key &key)
84 {
85 key.nr_samplers = info.base.file_max[TGSI_FILE_SAMPLER] + 1;
86
87 for (unsigned i = 0; i < key.nr_samplers; i++) {
88 if (info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
89 lp_sampler_static_sampler_state(
90 &key.sampler[i].sampler_state,
91 ctx->samplers[shader_type][i]);
92 }
93 }
94
95 /*
96 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
97 * are dx10-style? Can't really have mixed opcodes, at least not
98 * if we want to skip the holes here (without rescanning tgsi).
99 */
100 if (info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
101 key.nr_sampler_views =
102 info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
103 for (unsigned i = 0; i < key.nr_sampler_views; i++) {
104 if (info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1u << (i & 31))) {
105 const struct pipe_sampler_view *view =
106 ctx->sampler_views[shader_type][i];
107 lp_sampler_static_texture_state(
108 &key.sampler[i].texture_state, view);
109 if (view) {
110 struct swr_resource *swr_res = swr_resource(view->texture);
111 const struct util_format_description *desc =
112 util_format_description(view->format);
113 if (swr_res->has_depth && swr_res->has_stencil &&
114 !util_format_has_depth(desc))
115 key.sampler[i].texture_state.format = PIPE_FORMAT_S8_UINT;
116 }
117 }
118 }
119 } else {
120 key.nr_sampler_views = key.nr_samplers;
121 for (unsigned i = 0; i < key.nr_sampler_views; i++) {
122 if (info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
123 const struct pipe_sampler_view *view =
124 ctx->sampler_views[shader_type][i];
125 lp_sampler_static_texture_state(
126 &key.sampler[i].texture_state, view);
127 if (view) {
128 struct swr_resource *swr_res = swr_resource(view->texture);
129 const struct util_format_description *desc =
130 util_format_description(view->format);
131 if (swr_res->has_depth && swr_res->has_stencil &&
132 !util_format_has_depth(desc))
133 key.sampler[i].texture_state.format = PIPE_FORMAT_S8_UINT;
134 }
135 }
136 }
137 }
138 }
139
140 void
141 swr_generate_fs_key(struct swr_jit_fs_key &key,
142 struct swr_context *ctx,
143 swr_fragment_shader *swr_fs)
144 {
145 memset(&key, 0, sizeof(key));
146
147 key.nr_cbufs = ctx->framebuffer.nr_cbufs;
148 key.light_twoside = ctx->rasterizer->light_twoside;
149 key.sprite_coord_enable = ctx->rasterizer->sprite_coord_enable;
150
151 struct tgsi_shader_info *pPrevShader;
152 if (ctx->gs)
153 pPrevShader = &ctx->gs->info.base;
154 else
155 pPrevShader = &ctx->vs->info.base;
156
157 memcpy(&key.vs_output_semantic_name,
158 &pPrevShader->output_semantic_name,
159 sizeof(key.vs_output_semantic_name));
160 memcpy(&key.vs_output_semantic_idx,
161 &pPrevShader->output_semantic_index,
162 sizeof(key.vs_output_semantic_idx));
163
164 swr_generate_sampler_key(swr_fs->info, ctx, PIPE_SHADER_FRAGMENT, key);
165
166 key.poly_stipple_enable = ctx->rasterizer->poly_stipple_enable &&
167 ctx->poly_stipple.prim_is_poly;
168 }
169
170 void
171 swr_generate_vs_key(struct swr_jit_vs_key &key,
172 struct swr_context *ctx,
173 swr_vertex_shader *swr_vs)
174 {
175 memset(&key, 0, sizeof(key));
176
177 key.clip_plane_mask =
178 swr_vs->info.base.clipdist_writemask ?
179 swr_vs->info.base.clipdist_writemask & ctx->rasterizer->clip_plane_enable :
180 ctx->rasterizer->clip_plane_enable;
181
182 swr_generate_sampler_key(swr_vs->info, ctx, PIPE_SHADER_VERTEX, key);
183 }
184
185 void
186 swr_generate_fetch_key(struct swr_jit_fetch_key &key,
187 struct swr_vertex_element_state *velems)
188 {
189 memset(&key, 0, sizeof(key));
190
191 key.fsState = velems->fsState;
192 }
193
194 void
195 swr_generate_gs_key(struct swr_jit_gs_key &key,
196 struct swr_context *ctx,
197 swr_geometry_shader *swr_gs)
198 {
199 memset(&key, 0, sizeof(key));
200
201 struct tgsi_shader_info *pPrevShader = &ctx->vs->info.base;
202
203 memcpy(&key.vs_output_semantic_name,
204 &pPrevShader->output_semantic_name,
205 sizeof(key.vs_output_semantic_name));
206 memcpy(&key.vs_output_semantic_idx,
207 &pPrevShader->output_semantic_index,
208 sizeof(key.vs_output_semantic_idx));
209
210 swr_generate_sampler_key(swr_gs->info, ctx, PIPE_SHADER_GEOMETRY, key);
211 }
212
213 struct BuilderSWR : public Builder {
214 BuilderSWR(JitManager *pJitMgr, const char *pName)
215 : Builder(pJitMgr)
216 {
217 pJitMgr->SetupNewModule();
218 gallivm = gallivm_create(pName, wrap(&JM()->mContext));
219 pJitMgr->mpCurrentModule = unwrap(gallivm->module);
220 }
221
222 ~BuilderSWR() {
223 gallivm_free_ir(gallivm);
224 }
225
226 void WriteVS(Value *pVal, Value *pVsContext, Value *pVtxOutput,
227 unsigned slot, unsigned channel);
228
229 struct gallivm_state *gallivm;
230 PFN_VERTEX_FUNC CompileVS(struct swr_context *ctx, swr_jit_vs_key &key);
231 PFN_PIXEL_KERNEL CompileFS(struct swr_context *ctx, swr_jit_fs_key &key);
232 PFN_GS_FUNC CompileGS(struct swr_context *ctx, swr_jit_gs_key &key);
233
234 LLVMValueRef
235 swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
236 struct lp_build_tgsi_context * bld_base,
237 boolean is_vindex_indirect,
238 LLVMValueRef vertex_index,
239 boolean is_aindex_indirect,
240 LLVMValueRef attrib_index,
241 LLVMValueRef swizzle_index);
242 void
243 swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
244 struct lp_build_tgsi_context * bld_base,
245 LLVMValueRef (*outputs)[4],
246 LLVMValueRef emitted_vertices_vec);
247
248 void
249 swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
250 struct lp_build_tgsi_context * bld_base,
251 LLVMValueRef verts_per_prim_vec,
252 LLVMValueRef emitted_prims_vec);
253
254 void
255 swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
256 struct lp_build_tgsi_context * bld_base,
257 LLVMValueRef total_emitted_vertices_vec,
258 LLVMValueRef emitted_prims_vec);
259
260 };
261
262 struct swr_gs_llvm_iface {
263 struct lp_build_tgsi_gs_iface base;
264 struct tgsi_shader_info *info;
265
266 BuilderSWR *pBuilder;
267
268 Value *pGsCtx;
269 SWR_GS_STATE *pGsState;
270 uint32_t num_outputs;
271 uint32_t num_verts_per_prim;
272
273 Value *pVtxAttribMap;
274 };
275
276 // trampoline functions so we can use the builder llvm construction methods
277 static LLVMValueRef
278 swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
279 struct lp_build_tgsi_context * bld_base,
280 boolean is_vindex_indirect,
281 LLVMValueRef vertex_index,
282 boolean is_aindex_indirect,
283 LLVMValueRef attrib_index,
284 LLVMValueRef swizzle_index)
285 {
286 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_iface;
287
288 return iface->pBuilder->swr_gs_llvm_fetch_input(gs_iface, bld_base,
289 is_vindex_indirect,
290 vertex_index,
291 is_aindex_indirect,
292 attrib_index,
293 swizzle_index);
294 }
295
296 static void
297 swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
298 struct lp_build_tgsi_context * bld_base,
299 LLVMValueRef (*outputs)[4],
300 LLVMValueRef emitted_vertices_vec)
301 {
302 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
303
304 iface->pBuilder->swr_gs_llvm_emit_vertex(gs_base, bld_base,
305 outputs,
306 emitted_vertices_vec);
307 }
308
309 static void
310 swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
311 struct lp_build_tgsi_context * bld_base,
312 LLVMValueRef verts_per_prim_vec,
313 LLVMValueRef emitted_prims_vec)
314 {
315 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
316
317 iface->pBuilder->swr_gs_llvm_end_primitive(gs_base, bld_base,
318 verts_per_prim_vec,
319 emitted_prims_vec);
320 }
321
322 static void
323 swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
324 struct lp_build_tgsi_context * bld_base,
325 LLVMValueRef total_emitted_vertices_vec,
326 LLVMValueRef emitted_prims_vec)
327 {
328 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
329
330 iface->pBuilder->swr_gs_llvm_epilogue(gs_base, bld_base,
331 total_emitted_vertices_vec,
332 emitted_prims_vec);
333 }
334
335 LLVMValueRef
336 BuilderSWR::swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
337 struct lp_build_tgsi_context * bld_base,
338 boolean is_vindex_indirect,
339 LLVMValueRef vertex_index,
340 boolean is_aindex_indirect,
341 LLVMValueRef attrib_index,
342 LLVMValueRef swizzle_index)
343 {
344 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_iface;
345 Value *vert_index = unwrap(vertex_index);
346 Value *attr_index = unwrap(attrib_index);
347
348 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
349
350 if (is_vindex_indirect || is_aindex_indirect) {
351 int i;
352 Value *res = unwrap(bld_base->base.zero);
353 struct lp_type type = bld_base->base.type;
354
355 for (i = 0; i < type.length; i++) {
356 Value *vert_chan_index = vert_index;
357 Value *attr_chan_index = attr_index;
358
359 if (is_vindex_indirect) {
360 vert_chan_index = VEXTRACT(vert_index, C(i));
361 }
362 if (is_aindex_indirect) {
363 attr_chan_index = VEXTRACT(attr_index, C(i));
364 }
365
366 Value *attrib =
367 LOAD(GEP(iface->pVtxAttribMap, {C(0), attr_chan_index}));
368
369 Value *pVertex = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pVerts});
370 Value *pInputVertStride = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_inputVertStride});
371
372 Value *pVector = ADD(MUL(vert_chan_index, pInputVertStride), attrib);
373 Value *pInput = LOAD(GEP(pVertex, {pVector, unwrap(swizzle_index)}));
374
375 Value *value = VEXTRACT(pInput, C(i));
376 res = VINSERT(res, value, C(i));
377 }
378
379 return wrap(res);
380 } else {
381 Value *attrib = LOAD(GEP(iface->pVtxAttribMap, {C(0), attr_index}));
382
383 Value *pVertex = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pVerts});
384 Value *pInputVertStride = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_inputVertStride});
385
386 Value *pVector = ADD(MUL(vert_index, pInputVertStride), attrib);
387
388 Value *pInput = LOAD(GEP(pVertex, {pVector, unwrap(swizzle_index)}));
389
390 return wrap(pInput);
391 }
392 }
393
394 // GS output stream layout
395 #define VERTEX_COUNT_SIZE 32
396 #define CONTROL_HEADER_SIZE (8*32)
397
398 void
399 BuilderSWR::swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
400 struct lp_build_tgsi_context * bld_base,
401 LLVMValueRef (*outputs)[4],
402 LLVMValueRef emitted_vertices_vec)
403 {
404 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
405
406 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
407
408 const uint32_t headerSize = VERTEX_COUNT_SIZE + CONTROL_HEADER_SIZE;
409 const uint32_t attribSize = 4 * sizeof(float);
410 const uint32_t vertSize = attribSize * SWR_VTX_NUM_SLOTS;
411 Value *pVertexOffset = MUL(unwrap(emitted_vertices_vec), VIMMED1(vertSize));
412
413 Value *vMask = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_mask});
414 Value *vMask1 = TRUNC(vMask, VectorType::get(mInt1Ty, mVWidth));
415
416 Value *pStack = STACKSAVE();
417 Value *pTmpPtr = ALLOCA(mFP32Ty, C(4)); // used for dummy write for lane masking
418
419 for (uint32_t attrib = 0; attrib < iface->num_outputs; ++attrib) {
420 uint32_t attribSlot = attrib;
421 uint32_t sgvChannel = 0;
422 if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_PSIZE) {
423 attribSlot = VERTEX_SGV_SLOT;
424 sgvChannel = VERTEX_SGV_POINT_SIZE_COMP;
425 } else if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_LAYER) {
426 attribSlot = VERTEX_SGV_SLOT;
427 sgvChannel = VERTEX_SGV_RTAI_COMP;
428 } else if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_VIEWPORT_INDEX) {
429 attribSlot = VERTEX_SGV_SLOT;
430 sgvChannel = VERTEX_SGV_VAI_COMP;
431 } else if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_POSITION) {
432 attribSlot = VERTEX_POSITION_SLOT;
433 } else {
434 attribSlot = VERTEX_ATTRIB_START_SLOT + attrib;
435 if (iface->info->writes_position) {
436 attribSlot--;
437 }
438 }
439
440 Value *pOutputOffset = ADD(pVertexOffset, VIMMED1(headerSize + attribSize * attribSlot)); // + sgvChannel ?
441
442 for (uint32_t lane = 0; lane < mVWidth; ++lane) {
443 Value *pLaneOffset = VEXTRACT(pOutputOffset, C(lane));
444 Value *pStream = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
445 Value *pStreamOffset = GEP(pStream, pLaneOffset);
446 pStreamOffset = BITCAST(pStreamOffset, mFP32PtrTy);
447
448 Value *pLaneMask = VEXTRACT(vMask1, C(lane));
449 pStreamOffset = SELECT(pLaneMask, pStreamOffset, pTmpPtr);
450
451 for (uint32_t channel = 0; channel < 4; ++channel) {
452 Value *vData;
453
454 if (attribSlot == VERTEX_SGV_SLOT)
455 vData = LOAD(unwrap(outputs[attrib][0]));
456 else
457 vData = LOAD(unwrap(outputs[attrib][channel]));
458
459 if (attribSlot != VERTEX_SGV_SLOT ||
460 sgvChannel == channel) {
461 vData = VEXTRACT(vData, C(lane));
462 STORE(vData, pStreamOffset);
463 }
464 pStreamOffset = GEP(pStreamOffset, C(1));
465 }
466 }
467 }
468
469 STACKRESTORE(pStack);
470 }
471
472 void
473 BuilderSWR::swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
474 struct lp_build_tgsi_context * bld_base,
475 LLVMValueRef verts_per_prim_vec,
476 LLVMValueRef emitted_prims_vec)
477 {
478 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
479
480 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
481
482 Value *vMask = LOAD(iface->pGsCtx, { 0, SWR_GS_CONTEXT_mask });
483 Value *vMask1 = TRUNC(vMask, VectorType::get(mInt1Ty, 8));
484
485 uint32_t vertsPerPrim = iface->num_verts_per_prim;
486
487 Value *vCount =
488 ADD(MUL(unwrap(emitted_prims_vec), VIMMED1(vertsPerPrim)),
489 unwrap(verts_per_prim_vec));
490
491 struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
492 vCount = LOAD(unwrap(bld->total_emitted_vertices_vec_ptr));
493
494 struct lp_exec_mask *exec_mask = &bld->exec_mask;
495 Value *mask = unwrap(lp_build_mask_value(bld->mask));
496 if (exec_mask->has_mask)
497 mask = AND(mask, unwrap(exec_mask->exec_mask));
498
499 Value *cmpMask = VMASK(ICMP_NE(unwrap(verts_per_prim_vec), VIMMED1(0)));
500 mask = AND(mask, cmpMask);
501 vMask1 = TRUNC(mask, VectorType::get(mInt1Ty, 8));
502
503 vCount = SUB(vCount, VIMMED1(1));
504 Value *vOffset = ADD(UDIV(vCount, VIMMED1(8)), VIMMED1(VERTEX_COUNT_SIZE));
505 Value *vValue = SHL(VIMMED1(1), UREM(vCount, VIMMED1(8)));
506
507 vValue = TRUNC(vValue, VectorType::get(mInt8Ty, 8));
508
509 Value *pStack = STACKSAVE();
510 Value *pTmpPtr = ALLOCA(mInt8Ty, C(4)); // used for dummy read/write for lane masking
511
512 for (uint32_t lane = 0; lane < mVWidth; ++lane) {
513 Value *vLaneOffset = VEXTRACT(vOffset, C(lane));
514 Value *pStream = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
515 Value *pStreamOffset = GEP(pStream, vLaneOffset);
516
517 Value *pLaneMask = VEXTRACT(vMask1, C(lane));
518 pStreamOffset = SELECT(pLaneMask, pStreamOffset, pTmpPtr);
519
520 Value *vVal = LOAD(pStreamOffset);
521 vVal = OR(vVal, VEXTRACT(vValue, C(lane)));
522 STORE(vVal, pStreamOffset);
523 }
524
525 STACKRESTORE(pStack);
526 }
527
528 void
529 BuilderSWR::swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
530 struct lp_build_tgsi_context * bld_base,
531 LLVMValueRef total_emitted_vertices_vec,
532 LLVMValueRef emitted_prims_vec)
533 {
534 swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
535
536 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
537
538 // Store emit count to each output stream in the first DWORD
539 for (uint32_t lane = 0; lane < mVWidth; ++lane)
540 {
541 Value* pStream = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
542 pStream = BITCAST(pStream, mInt32PtrTy);
543 Value* pLaneCount = VEXTRACT(unwrap(total_emitted_vertices_vec), C(lane));
544 STORE(pLaneCount, pStream);
545 }
546 }
547
548 PFN_GS_FUNC
549 BuilderSWR::CompileGS(struct swr_context *ctx, swr_jit_gs_key &key)
550 {
551 SWR_GS_STATE *pGS = &ctx->gs->gsState;
552 struct tgsi_shader_info *info = &ctx->gs->info.base;
553
554 memset(pGS, 0, sizeof(*pGS));
555
556 pGS->gsEnable = true;
557
558 pGS->numInputAttribs = (VERTEX_ATTRIB_START_SLOT - VERTEX_POSITION_SLOT) + info->num_inputs;
559 pGS->outputTopology =
560 swr_convert_prim_topology(info->properties[TGSI_PROPERTY_GS_OUTPUT_PRIM]);
561 pGS->maxNumVerts = info->properties[TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES];
562 pGS->instanceCount = info->properties[TGSI_PROPERTY_GS_INVOCATIONS];
563
564 // XXX: single stream for now...
565 pGS->isSingleStream = true;
566 pGS->singleStreamID = 0;
567
568 pGS->vertexAttribOffset = VERTEX_POSITION_SLOT;
569 pGS->inputVertStride = pGS->numInputAttribs + pGS->vertexAttribOffset;
570 pGS->outputVertexSize = SWR_VTX_NUM_SLOTS;
571 pGS->controlDataSize = 8; // GS ouputs max of 8 32B units
572 pGS->controlDataOffset = VERTEX_COUNT_SIZE;
573 pGS->outputVertexOffset = pGS->controlDataOffset + CONTROL_HEADER_SIZE;
574
575 pGS->allocationSize =
576 VERTEX_COUNT_SIZE + // vertex count
577 CONTROL_HEADER_SIZE + // control header
578 (SWR_VTX_NUM_SLOTS * 16) * // sizeof vertex
579 pGS->maxNumVerts; // num verts
580
581 struct swr_geometry_shader *gs = ctx->gs;
582
583 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
584 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
585
586 memset(outputs, 0, sizeof(outputs));
587
588 AttrBuilder attrBuilder;
589 attrBuilder.addStackAlignmentAttr(JM()->mVWidth * sizeof(float));
590
591 std::vector<Type *> gsArgs{PointerType::get(Gen_swr_draw_context(JM()), 0),
592 PointerType::get(mInt8Ty, 0),
593 PointerType::get(Gen_SWR_GS_CONTEXT(JM()), 0)};
594 FunctionType *vsFuncType =
595 FunctionType::get(Type::getVoidTy(JM()->mContext), gsArgs, false);
596
597 // create new vertex shader function
598 auto pFunction = Function::Create(vsFuncType,
599 GlobalValue::ExternalLinkage,
600 "GS",
601 JM()->mpCurrentModule);
602 #if HAVE_LLVM < 0x0500
603 AttributeSet attrSet = AttributeSet::get(
604 JM()->mContext, AttributeSet::FunctionIndex, attrBuilder);
605 pFunction->addAttributes(AttributeSet::FunctionIndex, attrSet);
606 #else
607 pFunction->addAttributes(AttributeList::FunctionIndex, attrBuilder);
608 #endif
609
610 BasicBlock *block = BasicBlock::Create(JM()->mContext, "entry", pFunction);
611 IRB()->SetInsertPoint(block);
612 LLVMPositionBuilderAtEnd(gallivm->builder, wrap(block));
613
614 auto argitr = pFunction->arg_begin();
615 Value *hPrivateData = &*argitr++;
616 hPrivateData->setName("hPrivateData");
617 Value *pWorkerData = &*argitr++;
618 pWorkerData->setName("pWorkerData");
619 Value *pGsCtx = &*argitr++;
620 pGsCtx->setName("gsCtx");
621
622 Value *consts_ptr =
623 GEP(hPrivateData, {C(0), C(swr_draw_context_constantGS)});
624 consts_ptr->setName("gs_constants");
625 Value *const_sizes_ptr =
626 GEP(hPrivateData, {0, swr_draw_context_num_constantsGS});
627 const_sizes_ptr->setName("num_gs_constants");
628
629 struct lp_build_sampler_soa *sampler =
630 swr_sampler_soa_create(key.sampler, PIPE_SHADER_GEOMETRY);
631
632 struct lp_bld_tgsi_system_values system_values;
633 memset(&system_values, 0, sizeof(system_values));
634 system_values.prim_id = wrap(LOAD(pGsCtx, {0, SWR_GS_CONTEXT_PrimitiveID}));
635 system_values.instance_id = wrap(LOAD(pGsCtx, {0, SWR_GS_CONTEXT_InstanceID}));
636
637 std::vector<Constant*> mapConstants;
638 Value *vtxAttribMap = ALLOCA(ArrayType::get(mInt32Ty, PIPE_MAX_SHADER_INPUTS));
639 for (unsigned slot = 0; slot < info->num_inputs; slot++) {
640 ubyte semantic_name = info->input_semantic_name[slot];
641 ubyte semantic_idx = info->input_semantic_index[slot];
642
643 unsigned vs_slot = locate_linkage(semantic_name, semantic_idx, &ctx->vs->info.base);
644
645 vs_slot += VERTEX_ATTRIB_START_SLOT;
646
647 if (ctx->vs->info.base.output_semantic_name[0] == TGSI_SEMANTIC_POSITION)
648 vs_slot--;
649
650 if (semantic_name == TGSI_SEMANTIC_POSITION)
651 vs_slot = VERTEX_POSITION_SLOT;
652
653 STORE(C(vs_slot), vtxAttribMap, {0, slot});
654 mapConstants.push_back(C(vs_slot));
655 }
656
657 struct lp_build_mask_context mask;
658 Value *mask_val = LOAD(pGsCtx, {0, SWR_GS_CONTEXT_mask}, "gsMask");
659 lp_build_mask_begin(&mask, gallivm,
660 lp_type_float_vec(32, 32 * 8), wrap(mask_val));
661
662 // zero out cut buffer so we can load/modify/store bits
663 for (uint32_t lane = 0; lane < mVWidth; ++lane)
664 {
665 Value* pStream = LOAD(pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
666 MEMSET(pStream, C((char)0), VERTEX_COUNT_SIZE + CONTROL_HEADER_SIZE, sizeof(float) * KNOB_SIMD_WIDTH);
667 }
668
669 struct swr_gs_llvm_iface gs_iface;
670 gs_iface.base.fetch_input = ::swr_gs_llvm_fetch_input;
671 gs_iface.base.emit_vertex = ::swr_gs_llvm_emit_vertex;
672 gs_iface.base.end_primitive = ::swr_gs_llvm_end_primitive;
673 gs_iface.base.gs_epilogue = ::swr_gs_llvm_epilogue;
674 gs_iface.pBuilder = this;
675 gs_iface.pGsCtx = pGsCtx;
676 gs_iface.pGsState = pGS;
677 gs_iface.num_outputs = gs->info.base.num_outputs;
678 gs_iface.num_verts_per_prim =
679 u_vertices_per_prim((pipe_prim_type)info->properties[TGSI_PROPERTY_GS_OUTPUT_PRIM]);
680 gs_iface.info = info;
681 gs_iface.pVtxAttribMap = vtxAttribMap;
682
683 struct lp_build_tgsi_params params;
684 memset(&params, 0, sizeof(params));
685 params.type = lp_type_float_vec(32, 32 * 8);
686 params.mask = & mask;
687 params.consts_ptr = wrap(consts_ptr);
688 params.const_sizes_ptr = wrap(const_sizes_ptr);
689 params.system_values = &system_values;
690 params.inputs = inputs;
691 params.context_ptr = wrap(hPrivateData);
692 params.sampler = sampler;
693 params.info = &gs->info.base;
694 params.gs_iface = &gs_iface.base;
695
696 lp_build_tgsi_soa(gallivm,
697 gs->pipe.tokens,
698 &params,
699 outputs);
700
701 lp_build_mask_end(&mask);
702
703 sampler->destroy(sampler);
704
705 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
706
707 RET_VOID();
708
709 gallivm_verify_function(gallivm, wrap(pFunction));
710 gallivm_compile_module(gallivm);
711
712 PFN_GS_FUNC pFunc =
713 (PFN_GS_FUNC)gallivm_jit_function(gallivm, wrap(pFunction));
714
715 debug_printf("geom shader %p\n", pFunc);
716 assert(pFunc && "Error: GeomShader = NULL");
717
718 JM()->mIsModuleFinalized = true;
719
720 return pFunc;
721 }
722
723 PFN_GS_FUNC
724 swr_compile_gs(struct swr_context *ctx, swr_jit_gs_key &key)
725 {
726 BuilderSWR builder(
727 reinterpret_cast<JitManager *>(swr_screen(ctx->pipe.screen)->hJitMgr),
728 "GS");
729 PFN_GS_FUNC func = builder.CompileGS(ctx, key);
730
731 ctx->gs->map.insert(std::make_pair(key, std::make_unique<VariantGS>(builder.gallivm, func)));
732 return func;
733 }
734
735 void
736 BuilderSWR::WriteVS(Value *pVal, Value *pVsContext, Value *pVtxOutput, unsigned slot, unsigned channel)
737 {
738 #if USE_SIMD16_FRONTEND && !USE_SIMD16_VS
739 // interleave the simdvertex components into the dest simd16vertex
740 // slot16offset = slot8offset * 2
741 // comp16offset = comp8offset * 2 + alternateOffset
742
743 Value *offset = LOAD(pVsContext, { 0, SWR_VS_CONTEXT_AlternateOffset });
744 Value *pOut = GEP(pVtxOutput, { C(0), C(0), C(slot * 2), offset } );
745 STORE(pVal, pOut, {channel * 2});
746 #else
747 Value *pOut = GEP(pVtxOutput, {0, 0, slot});
748 STORE(pVal, pOut, {0, channel});
749 #endif
750 }
751
752 PFN_VERTEX_FUNC
753 BuilderSWR::CompileVS(struct swr_context *ctx, swr_jit_vs_key &key)
754 {
755 struct swr_vertex_shader *swr_vs = ctx->vs;
756
757 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
758 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
759
760 memset(outputs, 0, sizeof(outputs));
761
762 AttrBuilder attrBuilder;
763 attrBuilder.addStackAlignmentAttr(JM()->mVWidth * sizeof(float));
764
765 std::vector<Type *> vsArgs{PointerType::get(Gen_swr_draw_context(JM()), 0),
766 PointerType::get(mInt8Ty, 0),
767 PointerType::get(Gen_SWR_VS_CONTEXT(JM()), 0)};
768 FunctionType *vsFuncType =
769 FunctionType::get(Type::getVoidTy(JM()->mContext), vsArgs, false);
770
771 // create new vertex shader function
772 auto pFunction = Function::Create(vsFuncType,
773 GlobalValue::ExternalLinkage,
774 "VS",
775 JM()->mpCurrentModule);
776 #if HAVE_LLVM < 0x0500
777 AttributeSet attrSet = AttributeSet::get(
778 JM()->mContext, AttributeSet::FunctionIndex, attrBuilder);
779 pFunction->addAttributes(AttributeSet::FunctionIndex, attrSet);
780 #else
781 pFunction->addAttributes(AttributeList::FunctionIndex, attrBuilder);
782 #endif
783
784 BasicBlock *block = BasicBlock::Create(JM()->mContext, "entry", pFunction);
785 IRB()->SetInsertPoint(block);
786 LLVMPositionBuilderAtEnd(gallivm->builder, wrap(block));
787
788 auto argitr = pFunction->arg_begin();
789 Value *hPrivateData = &*argitr++;
790 hPrivateData->setName("hPrivateData");
791 Value *pWorkerData = &*argitr++;
792 pWorkerData->setName("pWorkerData");
793 Value *pVsCtx = &*argitr++;
794 pVsCtx->setName("vsCtx");
795
796 Value *consts_ptr = GEP(hPrivateData, {C(0), C(swr_draw_context_constantVS)});
797
798 consts_ptr->setName("vs_constants");
799 Value *const_sizes_ptr =
800 GEP(hPrivateData, {0, swr_draw_context_num_constantsVS});
801 const_sizes_ptr->setName("num_vs_constants");
802
803 Value *vtxInput = LOAD(pVsCtx, {0, SWR_VS_CONTEXT_pVin});
804 #if USE_SIMD16_VS
805 vtxInput = BITCAST(vtxInput, PointerType::get(Gen_simd16vertex(JM()), 0));
806 #endif
807
808 for (uint32_t attrib = 0; attrib < PIPE_MAX_SHADER_INPUTS; attrib++) {
809 const unsigned mask = swr_vs->info.base.input_usage_mask[attrib];
810 for (uint32_t channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
811 if (mask & (1 << channel)) {
812 inputs[attrib][channel] =
813 wrap(LOAD(vtxInput, {0, 0, attrib, channel}));
814 }
815 }
816 }
817
818 struct lp_build_sampler_soa *sampler =
819 swr_sampler_soa_create(key.sampler, PIPE_SHADER_VERTEX);
820
821 struct lp_bld_tgsi_system_values system_values;
822 memset(&system_values, 0, sizeof(system_values));
823 system_values.instance_id = wrap(LOAD(pVsCtx, {0, SWR_VS_CONTEXT_InstanceID}));
824
825 #if USE_SIMD16_VS
826 system_values.vertex_id = wrap(LOAD(pVsCtx, {0, SWR_VS_CONTEXT_VertexID16}));
827 #else
828 system_values.vertex_id = wrap(LOAD(pVsCtx, {0, SWR_VS_CONTEXT_VertexID}));
829 #endif
830
831 #if USE_SIMD16_VS
832 uint32_t vectorWidth = mVWidth16;
833 #else
834 uint32_t vectorWidth = mVWidth;
835 #endif
836
837 struct lp_build_tgsi_params params;
838 memset(&params, 0, sizeof(params));
839 params.type = lp_type_float_vec(32, 32 * vectorWidth);
840 params.consts_ptr = wrap(consts_ptr);
841 params.const_sizes_ptr = wrap(const_sizes_ptr);
842 params.system_values = &system_values;
843 params.inputs = inputs;
844 params.context_ptr = wrap(hPrivateData);
845 params.sampler = sampler;
846 params.info = &swr_vs->info.base;
847
848 lp_build_tgsi_soa(gallivm,
849 swr_vs->pipe.tokens,
850 &params,
851 outputs);
852
853 sampler->destroy(sampler);
854
855 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
856
857 Value *vtxOutput = LOAD(pVsCtx, {0, SWR_VS_CONTEXT_pVout});
858 #if USE_SIMD16_VS
859 vtxOutput = BITCAST(vtxOutput, PointerType::get(Gen_simd16vertex(JM()), 0));
860 #endif
861
862 for (uint32_t channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
863 for (uint32_t attrib = 0; attrib < PIPE_MAX_SHADER_OUTPUTS; attrib++) {
864 if (!outputs[attrib][channel])
865 continue;
866
867 Value *val;
868 uint32_t outSlot;
869
870 if (swr_vs->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_PSIZE) {
871 if (channel != VERTEX_SGV_POINT_SIZE_COMP)
872 continue;
873 val = LOAD(unwrap(outputs[attrib][0]));
874 outSlot = VERTEX_SGV_SLOT;
875 } else if (swr_vs->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_POSITION) {
876 val = LOAD(unwrap(outputs[attrib][channel]));
877 outSlot = VERTEX_POSITION_SLOT;
878 } else {
879 val = LOAD(unwrap(outputs[attrib][channel]));
880 outSlot = VERTEX_ATTRIB_START_SLOT + attrib;
881 if (swr_vs->info.base.output_semantic_name[0] == TGSI_SEMANTIC_POSITION)
882 outSlot--;
883 }
884
885 WriteVS(val, pVsCtx, vtxOutput, outSlot, channel);
886 }
887 }
888
889 if (ctx->rasterizer->clip_plane_enable ||
890 swr_vs->info.base.culldist_writemask) {
891 unsigned clip_mask = ctx->rasterizer->clip_plane_enable;
892
893 unsigned cv = 0;
894 if (swr_vs->info.base.writes_clipvertex) {
895 cv = locate_linkage(TGSI_SEMANTIC_CLIPVERTEX, 0,
896 &swr_vs->info.base);
897 } else {
898 for (int i = 0; i < PIPE_MAX_SHADER_OUTPUTS; i++) {
899 if (swr_vs->info.base.output_semantic_name[i] == TGSI_SEMANTIC_POSITION &&
900 swr_vs->info.base.output_semantic_index[i] == 0) {
901 cv = i;
902 break;
903 }
904 }
905 }
906 LLVMValueRef cx = LLVMBuildLoad(gallivm->builder, outputs[cv][0], "");
907 LLVMValueRef cy = LLVMBuildLoad(gallivm->builder, outputs[cv][1], "");
908 LLVMValueRef cz = LLVMBuildLoad(gallivm->builder, outputs[cv][2], "");
909 LLVMValueRef cw = LLVMBuildLoad(gallivm->builder, outputs[cv][3], "");
910
911 for (unsigned val = 0; val < PIPE_MAX_CLIP_PLANES; val++) {
912 // clip distance overrides user clip planes
913 if ((swr_vs->info.base.clipdist_writemask & clip_mask & (1 << val)) ||
914 ((swr_vs->info.base.culldist_writemask << swr_vs->info.base.num_written_clipdistance) & (1 << val))) {
915 unsigned cv = locate_linkage(TGSI_SEMANTIC_CLIPDIST, val < 4 ? 0 : 1,
916 &swr_vs->info.base);
917 if (val < 4) {
918 LLVMValueRef dist = LLVMBuildLoad(gallivm->builder, outputs[cv][val], "");
919 WriteVS(unwrap(dist), pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_LO_SLOT, val);
920 } else {
921 LLVMValueRef dist = LLVMBuildLoad(gallivm->builder, outputs[cv][val - 4], "");
922 WriteVS(unwrap(dist), pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_HI_SLOT, val - 4);
923 }
924 continue;
925 }
926
927 if (!(clip_mask & (1 << val)))
928 continue;
929
930 Value *px = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 0}));
931 Value *py = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 1}));
932 Value *pz = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 2}));
933 Value *pw = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 3}));
934 #if USE_SIMD16_VS
935 Value *bpx = VBROADCAST_16(px);
936 Value *bpy = VBROADCAST_16(py);
937 Value *bpz = VBROADCAST_16(pz);
938 Value *bpw = VBROADCAST_16(pw);
939 #else
940 Value *bpx = VBROADCAST(px);
941 Value *bpy = VBROADCAST(py);
942 Value *bpz = VBROADCAST(pz);
943 Value *bpw = VBROADCAST(pw);
944 #endif
945 Value *dist = FADD(FMUL(unwrap(cx), bpx),
946 FADD(FMUL(unwrap(cy), bpy),
947 FADD(FMUL(unwrap(cz), bpz),
948 FMUL(unwrap(cw), bpw))));
949
950 if (val < 4)
951 WriteVS(dist, pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_LO_SLOT, val);
952 else
953 WriteVS(dist, pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_HI_SLOT, val - 4);
954 }
955 }
956
957 RET_VOID();
958
959 gallivm_verify_function(gallivm, wrap(pFunction));
960 gallivm_compile_module(gallivm);
961
962 // lp_debug_dump_value(func);
963
964 PFN_VERTEX_FUNC pFunc =
965 (PFN_VERTEX_FUNC)gallivm_jit_function(gallivm, wrap(pFunction));
966
967 debug_printf("vert shader %p\n", pFunc);
968 assert(pFunc && "Error: VertShader = NULL");
969
970 JM()->mIsModuleFinalized = true;
971
972 return pFunc;
973 }
974
975 PFN_VERTEX_FUNC
976 swr_compile_vs(struct swr_context *ctx, swr_jit_vs_key &key)
977 {
978 if (!ctx->vs->pipe.tokens)
979 return NULL;
980
981 BuilderSWR builder(
982 reinterpret_cast<JitManager *>(swr_screen(ctx->pipe.screen)->hJitMgr),
983 "VS");
984 PFN_VERTEX_FUNC func = builder.CompileVS(ctx, key);
985
986 ctx->vs->map.insert(std::make_pair(key, std::make_unique<VariantVS>(builder.gallivm, func)));
987 return func;
988 }
989
990 unsigned
991 swr_so_adjust_attrib(unsigned in_attrib,
992 swr_vertex_shader *swr_vs)
993 {
994 ubyte semantic_name;
995 unsigned attrib;
996
997 attrib = in_attrib + VERTEX_ATTRIB_START_SLOT;
998
999 if (swr_vs) {
1000 semantic_name = swr_vs->info.base.output_semantic_name[in_attrib];
1001 if (semantic_name == TGSI_SEMANTIC_POSITION) {
1002 attrib = VERTEX_POSITION_SLOT;
1003 } else if (semantic_name == TGSI_SEMANTIC_PSIZE) {
1004 attrib = VERTEX_SGV_SLOT;
1005 } else if (semantic_name == TGSI_SEMANTIC_LAYER) {
1006 attrib = VERTEX_SGV_SLOT;
1007 } else {
1008 if (swr_vs->info.base.writes_position) {
1009 attrib--;
1010 }
1011 }
1012 }
1013
1014 return attrib;
1015 }
1016
1017 static unsigned
1018 locate_linkage(ubyte name, ubyte index, struct tgsi_shader_info *info)
1019 {
1020 for (int i = 0; i < PIPE_MAX_SHADER_OUTPUTS; i++) {
1021 if ((info->output_semantic_name[i] == name)
1022 && (info->output_semantic_index[i] == index)) {
1023 return i;
1024 }
1025 }
1026
1027 return 0xFFFFFFFF;
1028 }
1029
1030 PFN_PIXEL_KERNEL
1031 BuilderSWR::CompileFS(struct swr_context *ctx, swr_jit_fs_key &key)
1032 {
1033 struct swr_fragment_shader *swr_fs = ctx->fs;
1034
1035 struct tgsi_shader_info *pPrevShader;
1036 if (ctx->gs)
1037 pPrevShader = &ctx->gs->info.base;
1038 else
1039 pPrevShader = &ctx->vs->info.base;
1040
1041 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
1042 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
1043
1044 memset(inputs, 0, sizeof(inputs));
1045 memset(outputs, 0, sizeof(outputs));
1046
1047 struct lp_build_sampler_soa *sampler = NULL;
1048
1049 AttrBuilder attrBuilder;
1050 attrBuilder.addStackAlignmentAttr(JM()->mVWidth * sizeof(float));
1051
1052 std::vector<Type *> fsArgs{PointerType::get(Gen_swr_draw_context(JM()), 0),
1053 PointerType::get(mInt8Ty, 0),
1054 PointerType::get(Gen_SWR_PS_CONTEXT(JM()), 0)};
1055 FunctionType *funcType =
1056 FunctionType::get(Type::getVoidTy(JM()->mContext), fsArgs, false);
1057
1058 auto pFunction = Function::Create(funcType,
1059 GlobalValue::ExternalLinkage,
1060 "FS",
1061 JM()->mpCurrentModule);
1062 #if HAVE_LLVM < 0x0500
1063 AttributeSet attrSet = AttributeSet::get(
1064 JM()->mContext, AttributeSet::FunctionIndex, attrBuilder);
1065 pFunction->addAttributes(AttributeSet::FunctionIndex, attrSet);
1066 #else
1067 pFunction->addAttributes(AttributeList::FunctionIndex, attrBuilder);
1068 #endif
1069
1070 BasicBlock *block = BasicBlock::Create(JM()->mContext, "entry", pFunction);
1071 IRB()->SetInsertPoint(block);
1072 LLVMPositionBuilderAtEnd(gallivm->builder, wrap(block));
1073
1074 auto args = pFunction->arg_begin();
1075 Value *hPrivateData = &*args++;
1076 hPrivateData->setName("hPrivateData");
1077 Value *pWorkerData = &*args++;
1078 pWorkerData->setName("pWorkerData");
1079 Value *pPS = &*args++;
1080 pPS->setName("psCtx");
1081
1082 Value *consts_ptr = GEP(hPrivateData, {0, swr_draw_context_constantFS});
1083 consts_ptr->setName("fs_constants");
1084 Value *const_sizes_ptr =
1085 GEP(hPrivateData, {0, swr_draw_context_num_constantsFS});
1086 const_sizes_ptr->setName("num_fs_constants");
1087
1088 // load *pAttribs, *pPerspAttribs
1089 Value *pRawAttribs = LOAD(pPS, {0, SWR_PS_CONTEXT_pAttribs}, "pRawAttribs");
1090 Value *pPerspAttribs =
1091 LOAD(pPS, {0, SWR_PS_CONTEXT_pPerspAttribs}, "pPerspAttribs");
1092
1093 swr_fs->constantMask = 0;
1094 swr_fs->flatConstantMask = 0;
1095 swr_fs->pointSpriteMask = 0;
1096
1097 for (int attrib = 0; attrib < PIPE_MAX_SHADER_INPUTS; attrib++) {
1098 const unsigned mask = swr_fs->info.base.input_usage_mask[attrib];
1099 const unsigned interpMode = swr_fs->info.base.input_interpolate[attrib];
1100 const unsigned interpLoc = swr_fs->info.base.input_interpolate_loc[attrib];
1101
1102 if (!mask)
1103 continue;
1104
1105 // load i,j
1106 Value *vi = nullptr, *vj = nullptr;
1107 switch (interpLoc) {
1108 case TGSI_INTERPOLATE_LOC_CENTER:
1109 vi = LOAD(pPS, {0, SWR_PS_CONTEXT_vI, PixelPositions_center}, "i");
1110 vj = LOAD(pPS, {0, SWR_PS_CONTEXT_vJ, PixelPositions_center}, "j");
1111 break;
1112 case TGSI_INTERPOLATE_LOC_CENTROID:
1113 vi = LOAD(pPS, {0, SWR_PS_CONTEXT_vI, PixelPositions_centroid}, "i");
1114 vj = LOAD(pPS, {0, SWR_PS_CONTEXT_vJ, PixelPositions_centroid}, "j");
1115 break;
1116 case TGSI_INTERPOLATE_LOC_SAMPLE:
1117 vi = LOAD(pPS, {0, SWR_PS_CONTEXT_vI, PixelPositions_sample}, "i");
1118 vj = LOAD(pPS, {0, SWR_PS_CONTEXT_vJ, PixelPositions_sample}, "j");
1119 break;
1120 }
1121
1122 // load/compute w
1123 Value *vw = nullptr, *pAttribs;
1124 if (interpMode == TGSI_INTERPOLATE_PERSPECTIVE ||
1125 interpMode == TGSI_INTERPOLATE_COLOR) {
1126 pAttribs = pPerspAttribs;
1127 switch (interpLoc) {
1128 case TGSI_INTERPOLATE_LOC_CENTER:
1129 vw = VRCP(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_center}));
1130 break;
1131 case TGSI_INTERPOLATE_LOC_CENTROID:
1132 vw = VRCP(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_centroid}));
1133 break;
1134 case TGSI_INTERPOLATE_LOC_SAMPLE:
1135 vw = VRCP(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_sample}));
1136 break;
1137 }
1138 } else {
1139 pAttribs = pRawAttribs;
1140 vw = VIMMED1(1.f);
1141 }
1142
1143 vw->setName("w");
1144
1145 ubyte semantic_name = swr_fs->info.base.input_semantic_name[attrib];
1146 ubyte semantic_idx = swr_fs->info.base.input_semantic_index[attrib];
1147
1148 if (semantic_name == TGSI_SEMANTIC_FACE) {
1149 Value *ff =
1150 UI_TO_FP(LOAD(pPS, {0, SWR_PS_CONTEXT_frontFace}), mFP32Ty);
1151 ff = FSUB(FMUL(ff, C(2.0f)), C(1.0f));
1152 ff = VECTOR_SPLAT(JM()->mVWidth, ff, "vFrontFace");
1153
1154 inputs[attrib][0] = wrap(ff);
1155 inputs[attrib][1] = wrap(VIMMED1(0.0f));
1156 inputs[attrib][2] = wrap(VIMMED1(0.0f));
1157 inputs[attrib][3] = wrap(VIMMED1(1.0f));
1158 continue;
1159 } else if (semantic_name == TGSI_SEMANTIC_POSITION) { // gl_FragCoord
1160 if (swr_fs->info.base.properties[TGSI_PROPERTY_FS_COORD_PIXEL_CENTER] ==
1161 TGSI_FS_COORD_PIXEL_CENTER_HALF_INTEGER) {
1162 inputs[attrib][0] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vX, PixelPositions_center}, "vX"));
1163 inputs[attrib][1] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vY, PixelPositions_center}, "vY"));
1164 } else {
1165 inputs[attrib][0] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vX, PixelPositions_UL}, "vX"));
1166 inputs[attrib][1] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vY, PixelPositions_UL}, "vY"));
1167 }
1168 inputs[attrib][2] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vZ}, "vZ"));
1169 inputs[attrib][3] =
1170 wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_center}, "vOneOverW"));
1171 continue;
1172 } else if (semantic_name == TGSI_SEMANTIC_LAYER) { // gl_Layer
1173 Value *ff = LOAD(pPS, {0, SWR_PS_CONTEXT_renderTargetArrayIndex});
1174 ff = VECTOR_SPLAT(JM()->mVWidth, ff, "vRenderTargetArrayIndex");
1175 inputs[attrib][0] = wrap(ff);
1176 inputs[attrib][1] = wrap(VIMMED1(0.0f));
1177 inputs[attrib][2] = wrap(VIMMED1(0.0f));
1178 inputs[attrib][3] = wrap(VIMMED1(0.0f));
1179 continue;
1180 } else if (semantic_name == TGSI_SEMANTIC_VIEWPORT_INDEX) { // gl_ViewportIndex
1181 Value *ff = LOAD(pPS, {0, SWR_PS_CONTEXT_viewportIndex});
1182 ff = VECTOR_SPLAT(JM()->mVWidth, ff, "vViewportIndex");
1183 inputs[attrib][0] = wrap(ff);
1184 inputs[attrib][1] = wrap(VIMMED1(0.0f));
1185 inputs[attrib][2] = wrap(VIMMED1(0.0f));
1186 inputs[attrib][3] = wrap(VIMMED1(0.0f));
1187 continue;
1188 }
1189 unsigned linkedAttrib =
1190 locate_linkage(semantic_name, semantic_idx, pPrevShader) - 1;
1191
1192 uint32_t extraAttribs = 0;
1193 if (semantic_name == TGSI_SEMANTIC_PRIMID && !ctx->gs) {
1194 /* non-gs generated primID - need to grab from swizzleMap override */
1195 linkedAttrib = pPrevShader->num_outputs - 1;
1196 swr_fs->constantMask |= 1 << linkedAttrib;
1197 extraAttribs++;
1198 } else if (semantic_name == TGSI_SEMANTIC_GENERIC &&
1199 key.sprite_coord_enable & (1 << semantic_idx)) {
1200 /* we add an extra attrib to the backendState in swr_update_derived. */
1201 linkedAttrib = pPrevShader->num_outputs + extraAttribs - 1;
1202 swr_fs->pointSpriteMask |= (1 << linkedAttrib);
1203 extraAttribs++;
1204 } else if (linkedAttrib == 0xFFFFFFFF) {
1205 inputs[attrib][0] = wrap(VIMMED1(0.0f));
1206 inputs[attrib][1] = wrap(VIMMED1(0.0f));
1207 inputs[attrib][2] = wrap(VIMMED1(0.0f));
1208 inputs[attrib][3] = wrap(VIMMED1(1.0f));
1209 /* If we're reading in color and 2-sided lighting is enabled, we have
1210 * to keep going.
1211 */
1212 if (semantic_name != TGSI_SEMANTIC_COLOR || !key.light_twoside)
1213 continue;
1214 } else {
1215 if (interpMode == TGSI_INTERPOLATE_CONSTANT) {
1216 swr_fs->constantMask |= 1 << linkedAttrib;
1217 } else if (interpMode == TGSI_INTERPOLATE_COLOR) {
1218 swr_fs->flatConstantMask |= 1 << linkedAttrib;
1219 }
1220 }
1221
1222 unsigned bcolorAttrib = 0xFFFFFFFF;
1223 Value *offset = NULL;
1224 if (semantic_name == TGSI_SEMANTIC_COLOR && key.light_twoside) {
1225 bcolorAttrib = locate_linkage(
1226 TGSI_SEMANTIC_BCOLOR, semantic_idx, pPrevShader) - 1;
1227 /* Neither front nor back colors were available. Nothing to load. */
1228 if (bcolorAttrib == 0xFFFFFFFF && linkedAttrib == 0xFFFFFFFF)
1229 continue;
1230 /* If there is no front color, just always use the back color. */
1231 if (linkedAttrib == 0xFFFFFFFF)
1232 linkedAttrib = bcolorAttrib;
1233
1234 if (bcolorAttrib != 0xFFFFFFFF) {
1235 if (interpMode == TGSI_INTERPOLATE_CONSTANT) {
1236 swr_fs->constantMask |= 1 << bcolorAttrib;
1237 } else if (interpMode == TGSI_INTERPOLATE_COLOR) {
1238 swr_fs->flatConstantMask |= 1 << bcolorAttrib;
1239 }
1240
1241 unsigned diff = 12 * (bcolorAttrib - linkedAttrib);
1242
1243 if (diff) {
1244 Value *back =
1245 XOR(C(1), LOAD(pPS, {0, SWR_PS_CONTEXT_frontFace}), "backFace");
1246
1247 offset = MUL(back, C(diff));
1248 offset->setName("offset");
1249 }
1250 }
1251 }
1252
1253 for (int channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
1254 if (mask & (1 << channel)) {
1255 Value *indexA = C(linkedAttrib * 12 + channel);
1256 Value *indexB = C(linkedAttrib * 12 + channel + 4);
1257 Value *indexC = C(linkedAttrib * 12 + channel + 8);
1258
1259 if (offset) {
1260 indexA = ADD(indexA, offset);
1261 indexB = ADD(indexB, offset);
1262 indexC = ADD(indexC, offset);
1263 }
1264
1265 Value *va = VBROADCAST(LOAD(GEP(pAttribs, indexA)));
1266 Value *vb = VBROADCAST(LOAD(GEP(pAttribs, indexB)));
1267 Value *vc = VBROADCAST(LOAD(GEP(pAttribs, indexC)));
1268
1269 if (interpMode == TGSI_INTERPOLATE_CONSTANT) {
1270 inputs[attrib][channel] = wrap(va);
1271 } else {
1272 Value *vk = FSUB(FSUB(VIMMED1(1.0f), vi), vj);
1273
1274 vc = FMUL(vk, vc);
1275
1276 Value *interp = FMUL(va, vi);
1277 Value *interp1 = FMUL(vb, vj);
1278 interp = FADD(interp, interp1);
1279 interp = FADD(interp, vc);
1280 if (interpMode == TGSI_INTERPOLATE_PERSPECTIVE ||
1281 interpMode == TGSI_INTERPOLATE_COLOR)
1282 interp = FMUL(interp, vw);
1283 inputs[attrib][channel] = wrap(interp);
1284 }
1285 }
1286 }
1287 }
1288
1289 sampler = swr_sampler_soa_create(key.sampler, PIPE_SHADER_FRAGMENT);
1290
1291 struct lp_bld_tgsi_system_values system_values;
1292 memset(&system_values, 0, sizeof(system_values));
1293
1294 struct lp_build_mask_context mask;
1295 bool uses_mask = false;
1296
1297 if (swr_fs->info.base.uses_kill ||
1298 key.poly_stipple_enable) {
1299 Value *vActiveMask = NULL;
1300 if (swr_fs->info.base.uses_kill) {
1301 vActiveMask = LOAD(pPS, {0, SWR_PS_CONTEXT_activeMask}, "activeMask");
1302 }
1303 if (key.poly_stipple_enable) {
1304 // first get fragment xy coords and clip to stipple bounds
1305 Value *vXf = LOAD(pPS, {0, SWR_PS_CONTEXT_vX, PixelPositions_UL});
1306 Value *vYf = LOAD(pPS, {0, SWR_PS_CONTEXT_vY, PixelPositions_UL});
1307 Value *vXu = FP_TO_UI(vXf, mSimdInt32Ty);
1308 Value *vYu = FP_TO_UI(vYf, mSimdInt32Ty);
1309
1310 // stipple pattern is 32x32, which means that one line of stipple
1311 // is stored in one word:
1312 // vXstipple is bit offset inside 32-bit stipple word
1313 // vYstipple is word index is stipple array
1314 Value *vXstipple = AND(vXu, VIMMED1(0x1f)); // & (32-1)
1315 Value *vYstipple = AND(vYu, VIMMED1(0x1f)); // & (32-1)
1316
1317 // grab stipple pattern base address
1318 Value *stipplePtr = GEP(hPrivateData, {0, swr_draw_context_polyStipple, 0});
1319 stipplePtr = BITCAST(stipplePtr, mInt8PtrTy);
1320
1321 // peform a gather to grab stipple words for each lane
1322 Value *vStipple = GATHERDD(VUNDEF_I(), stipplePtr, vYstipple,
1323 VIMMED1(0xffffffff), 4);
1324
1325 // create a mask with one bit corresponding to the x stipple
1326 // and AND it with the pattern, to see if we have a bit
1327 Value *vBitMask = LSHR(VIMMED1(0x80000000), vXstipple);
1328 Value *vStippleMask = AND(vStipple, vBitMask);
1329 vStippleMask = ICMP_NE(vStippleMask, VIMMED1(0));
1330 vStippleMask = VMASK(vStippleMask);
1331
1332 if (swr_fs->info.base.uses_kill) {
1333 vActiveMask = AND(vActiveMask, vStippleMask);
1334 } else {
1335 vActiveMask = vStippleMask;
1336 }
1337 }
1338 lp_build_mask_begin(
1339 &mask, gallivm, lp_type_float_vec(32, 32 * 8), wrap(vActiveMask));
1340 uses_mask = true;
1341 }
1342
1343 struct lp_build_tgsi_params params;
1344 memset(&params, 0, sizeof(params));
1345 params.type = lp_type_float_vec(32, 32 * 8);
1346 params.mask = uses_mask ? &mask : NULL;
1347 params.consts_ptr = wrap(consts_ptr);
1348 params.const_sizes_ptr = wrap(const_sizes_ptr);
1349 params.system_values = &system_values;
1350 params.inputs = inputs;
1351 params.context_ptr = wrap(hPrivateData);
1352 params.sampler = sampler;
1353 params.info = &swr_fs->info.base;
1354
1355 lp_build_tgsi_soa(gallivm,
1356 swr_fs->pipe.tokens,
1357 &params,
1358 outputs);
1359
1360 sampler->destroy(sampler);
1361
1362 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
1363
1364 for (uint32_t attrib = 0; attrib < swr_fs->info.base.num_outputs;
1365 attrib++) {
1366 switch (swr_fs->info.base.output_semantic_name[attrib]) {
1367 case TGSI_SEMANTIC_POSITION: {
1368 // write z
1369 LLVMValueRef outZ =
1370 LLVMBuildLoad(gallivm->builder, outputs[attrib][2], "");
1371 STORE(unwrap(outZ), pPS, {0, SWR_PS_CONTEXT_vZ});
1372 break;
1373 }
1374 case TGSI_SEMANTIC_COLOR: {
1375 for (uint32_t channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
1376 if (!outputs[attrib][channel])
1377 continue;
1378
1379 LLVMValueRef out =
1380 LLVMBuildLoad(gallivm->builder, outputs[attrib][channel], "");
1381 if (swr_fs->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS] &&
1382 swr_fs->info.base.output_semantic_index[attrib] == 0) {
1383 for (uint32_t rt = 0; rt < key.nr_cbufs; rt++) {
1384 STORE(unwrap(out),
1385 pPS,
1386 {0, SWR_PS_CONTEXT_shaded, rt, channel});
1387 }
1388 } else {
1389 STORE(unwrap(out),
1390 pPS,
1391 {0,
1392 SWR_PS_CONTEXT_shaded,
1393 swr_fs->info.base.output_semantic_index[attrib],
1394 channel});
1395 }
1396 }
1397 break;
1398 }
1399 default: {
1400 fprintf(stderr,
1401 "unknown output from FS %s[%d]\n",
1402 tgsi_semantic_names[swr_fs->info.base
1403 .output_semantic_name[attrib]],
1404 swr_fs->info.base.output_semantic_index[attrib]);
1405 break;
1406 }
1407 }
1408 }
1409
1410 LLVMValueRef mask_result = 0;
1411 if (uses_mask) {
1412 mask_result = lp_build_mask_end(&mask);
1413 }
1414
1415 IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
1416
1417 if (uses_mask) {
1418 STORE(unwrap(mask_result), pPS, {0, SWR_PS_CONTEXT_activeMask});
1419 }
1420
1421 RET_VOID();
1422
1423 gallivm_verify_function(gallivm, wrap(pFunction));
1424
1425 gallivm_compile_module(gallivm);
1426
1427 // after the gallivm passes, we have to lower the core's intrinsics
1428 llvm::legacy::FunctionPassManager lowerPass(JM()->mpCurrentModule);
1429 lowerPass.add(createLowerX86Pass(this));
1430 lowerPass.run(*pFunction);
1431
1432 PFN_PIXEL_KERNEL kernel =
1433 (PFN_PIXEL_KERNEL)gallivm_jit_function(gallivm, wrap(pFunction));
1434 debug_printf("frag shader %p\n", kernel);
1435 assert(kernel && "Error: FragShader = NULL");
1436
1437 JM()->mIsModuleFinalized = true;
1438
1439 return kernel;
1440 }
1441
1442 PFN_PIXEL_KERNEL
1443 swr_compile_fs(struct swr_context *ctx, swr_jit_fs_key &key)
1444 {
1445 if (!ctx->fs->pipe.tokens)
1446 return NULL;
1447
1448 BuilderSWR builder(
1449 reinterpret_cast<JitManager *>(swr_screen(ctx->pipe.screen)->hJitMgr),
1450 "FS");
1451 PFN_PIXEL_KERNEL func = builder.CompileFS(ctx, key);
1452
1453 ctx->fs->map.insert(std::make_pair(key, std::make_unique<VariantFS>(builder.gallivm, func)));
1454 return func;
1455 }